Initial commit

This commit is contained in:
shiyu
2026-02-09 13:19:28 +08:00
commit 17d13999b0
300 changed files with 48565 additions and 0 deletions
+24
View File
@@ -0,0 +1,24 @@
from .providers import BaseAdapter
from .registry import (
RuntimeRegistry,
discover_adapters,
get_config_schema,
get_config_schemas,
normalize_adapter_type,
runtime_registry,
)
from .service import AdapterService
from .types import AdapterCreate, AdapterOut
__all__ = [
"BaseAdapter",
"RuntimeRegistry",
"discover_adapters",
"get_config_schema",
"get_config_schemas",
"normalize_adapter_type",
"runtime_registry",
"AdapterService",
"AdapterCreate",
"AdapterOut",
]
+92
View File
@@ -0,0 +1,92 @@
from typing import Annotated
from fastapi import APIRouter, Depends, Request
from api.response import success
from domain.audit import AuditAction, audit
from domain.auth import User, get_current_active_user
from domain.permission import require_system_permission
from domain.permission.types import AdapterPermission
from .service import AdapterService
from .types import AdapterCreate
router = APIRouter(prefix="/api/adapters", tags=["adapters"])
@router.post("")
@audit(
action=AuditAction.CREATE,
description="创建存储适配器",
body_fields=["name", "type", "path", "sub_path", "enabled"],
)
@require_system_permission(AdapterPermission.CREATE)
async def create_adapter(
request: Request,
data: AdapterCreate,
current_user: Annotated[User, Depends(get_current_active_user)]
):
adapter = await AdapterService.create_adapter(data, current_user)
return success(adapter)
@router.get("")
@audit(action=AuditAction.READ, description="获取适配器列表")
@require_system_permission(AdapterPermission.LIST)
async def list_adapters(
request: Request,
current_user: Annotated[User, Depends(get_current_active_user)]
):
adapters = await AdapterService.list_adapters()
return success(adapters)
@router.get("/available")
@audit(action=AuditAction.READ, description="获取可用适配器类型")
@require_system_permission(AdapterPermission.LIST)
async def available_adapter_types(
request: Request,
current_user: Annotated[User, Depends(get_current_active_user)]
):
data = await AdapterService.available_adapter_types()
return success(data)
@router.get("/{adapter_id}")
@audit(action=AuditAction.READ, description="获取适配器详情")
@require_system_permission(AdapterPermission.LIST)
async def get_adapter(
request: Request,
adapter_id: int,
current_user: Annotated[User, Depends(get_current_active_user)]
):
adapter = await AdapterService.get_adapter(adapter_id)
return success(adapter)
@router.put("/{adapter_id}")
@audit(
action=AuditAction.UPDATE,
description="更新存储适配器",
body_fields=["name", "type", "path", "sub_path", "enabled"],
)
@require_system_permission(AdapterPermission.EDIT)
async def update_adapter(
request: Request,
adapter_id: int,
data: AdapterCreate,
current_user: Annotated[User, Depends(get_current_active_user)]
):
adapter = await AdapterService.update_adapter(adapter_id, data, current_user)
return success(adapter)
@router.delete("/{adapter_id}")
@audit(action=AuditAction.DELETE, description="删除存储适配器")
@require_system_permission(AdapterPermission.DELETE)
async def delete_adapter(
request: Request,
adapter_id: int,
current_user: Annotated[User, Depends(get_current_active_user)]
):
result = await AdapterService.delete_adapter(adapter_id, current_user)
return success(result)
+3
View File
@@ -0,0 +1,3 @@
from .base import BaseAdapter
__all__ = ["BaseAdapter"]
+515
View File
@@ -0,0 +1,515 @@
import asyncio
import mimetypes
import re
import tempfile
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, AsyncIterator, Dict, List, Tuple
from urllib.parse import quote, urljoin
import httpx
from fastapi import HTTPException
from fastapi.responses import Response, StreamingResponse
from models import StorageAdapter
def _normalize_fs_path(path: str) -> str:
path = (path or "").replace("\\", "/").strip()
if not path or path == "/":
return "/"
if not path.startswith("/"):
path = "/" + path
path = re.sub(r"/{2,}", "/", path)
if path != "/" and path.endswith("/"):
path = path.rstrip("/")
return path or "/"
def _join_fs_path(base: str, rel: str) -> str:
base = _normalize_fs_path(base)
rel = (rel or "").replace("\\", "/").lstrip("/")
if not rel:
return base
if base == "/":
return "/" + rel
return f"{base}/{rel}"
def _split_parent_and_name(path: str) -> Tuple[str, str]:
path = _normalize_fs_path(path)
if path == "/":
return "/", ""
parent, _, name = path.rpartition("/")
if not parent:
parent = "/"
return parent, name
def _parse_iso_to_epoch(value: str | None) -> int:
if not value:
return 0
text = str(value).strip()
if not text:
return 0
try:
if text.endswith("Z"):
text = text[:-1] + "+00:00"
m = re.match(r"^(.*?)(\.\d+)([+-]\d\d:\d\d)?$", text)
if m:
head, frac, tz = m.group(1), m.group(2), m.group(3) or ""
digits = frac[1:]
if len(digits) > 6:
frac = "." + digits[:6]
text = head + frac + tz
dt = datetime.fromisoformat(text)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return int(dt.timestamp())
except Exception:
return 0
class AListApiAdapterBase:
def __init__(self, record: StorageAdapter, *, product_name: str):
self.record = record
self.product_name = product_name
cfg = record.config or {}
self.base_url: str = str(cfg.get("base_url", "")).rstrip("/")
if not self.base_url.startswith("http"):
raise ValueError(f"{product_name} requires base_url http/https")
self.username: str = str(cfg.get("username") or "")
self.password: str = str(cfg.get("password") or "")
if (self.username and not self.password) or (self.password and not self.username):
raise ValueError(f"{product_name} requires both username and password")
self.use_auth: bool = bool(self.username and self.password)
self.timeout: float = float(cfg.get("timeout", 30))
self.root_path: str = _normalize_fs_path(str(cfg.get("root") or "/"))
self.enable_redirect_307: bool = bool(cfg.get("enable_direct_download_307"))
self._token: str | None = None
self._login_lock = asyncio.Lock()
def get_effective_root(self, sub_path: str | None) -> str:
base = _normalize_fs_path(self.root_path)
if sub_path:
return _join_fs_path(base, sub_path)
return base
async def _ensure_token(self) -> str:
if not self.use_auth:
return ""
if self._token:
return self._token
async with self._login_lock:
if self._token:
return self._token
self._token = await self._login()
return self._token
async def _login(self) -> str:
url = self.base_url + "/api/auth/login"
body = {"username": self.username, "password": self.password}
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.post(url, json=body)
resp.raise_for_status()
payload = resp.json()
if not isinstance(payload, dict):
raise HTTPException(502, detail=f"{self.product_name} login: invalid response")
code = payload.get("code")
if code not in (0, 200):
raise HTTPException(502, detail=f"{self.product_name} login failed: {payload.get('message')}")
data = payload.get("data") or {}
token = (data.get("token") if isinstance(data, dict) else None) or ""
token = str(token).strip()
if not token:
raise HTTPException(502, detail=f"{self.product_name} login: missing token")
return token
async def _api_json(
self,
method: str,
endpoint: str,
*,
json: Dict[str, Any] | None = None,
headers: Dict[str, str] | None = None,
retry: bool = True,
files: Any = None,
) -> Any:
token = await self._ensure_token()
url = self.base_url + endpoint
req_headers: Dict[str, str] = {}
if token:
req_headers["Authorization"] = token
if headers:
req_headers.update(headers)
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.request(method, url, json=json, headers=req_headers, files=files)
if resp.status_code == 401 and retry and self.use_auth:
self._token = None
return await self._api_json(method, endpoint, json=json, headers=headers, retry=False, files=files)
resp.raise_for_status()
payload = resp.json()
if not isinstance(payload, dict):
raise HTTPException(502, detail=f"{self.product_name} api: invalid response")
code = payload.get("code")
if code in (0, 200):
return payload.get("data")
if code in (401, 403) and retry and self.use_auth:
self._token = None
return await self._api_json(method, endpoint, json=json, headers=headers, retry=False, files=files)
if code == 404:
raise FileNotFoundError(json.get("path") if json else "")
msg = payload.get("message") or payload.get("msg") or ""
raise HTTPException(502, detail=f"{self.product_name} api error code={code} msg={msg}")
def _abs_url(self, url: str) -> str:
u = (url or "").strip()
if not u:
return ""
if u.startswith("http://") or u.startswith("https://"):
return u
return urljoin(self.base_url.rstrip("/") + "/", u.lstrip("/"))
async def _fs_list(self, path: str) -> Dict[str, Any]:
body = {"path": path, "password": "", "page": 1, "per_page": 0, "refresh": False}
data = await self._api_json("POST", "/api/fs/list", json=body)
return data or {}
async def _fs_get(self, path: str) -> Dict[str, Any]:
body = {"path": path, "password": "", "page": 1, "per_page": 0, "refresh": False}
data = await self._api_json("POST", "/api/fs/get", json=body)
return data or {}
async def list_dir(
self,
root: str,
rel: str,
page_num: int = 1,
page_size: int = 50,
sort_by: str = "name",
sort_order: str = "asc",
) -> Tuple[List[Dict], int]:
path = _join_fs_path(root, rel)
data = await self._fs_list(path)
content = data.get("content") or []
if not isinstance(content, list):
raise HTTPException(502, detail=f"{self.product_name} list_dir: invalid content")
entries: List[Dict] = []
for it in content:
if not isinstance(it, dict):
continue
name = str(it.get("name") or "")
if not name:
continue
is_dir = bool(it.get("is_dir"))
size = int(it.get("size") or 0) if not is_dir else 0
mtime = _parse_iso_to_epoch(it.get("modified"))
entries.append(
{
"name": name,
"is_dir": is_dir,
"size": size,
"mtime": mtime,
"type": "dir" if is_dir else "file",
}
)
reverse = sort_order.lower() == "desc"
def get_sort_key(item: Dict) -> Tuple:
key = (not item.get("is_dir"),)
f = sort_by.lower()
if f == "name":
key += (str(item.get("name", "")).lower(),)
elif f == "size":
key += (int(item.get("size", 0)),)
elif f == "mtime":
key += (int(item.get("mtime", 0)),)
else:
key += (str(item.get("name", "")).lower(),)
return key
entries.sort(key=get_sort_key, reverse=reverse)
total = len(entries)
start = (page_num - 1) * page_size
end = start + page_size
return entries[start:end], total
async def stat_file(self, root: str, rel: str):
path = _join_fs_path(root, rel)
data = await self._fs_get(path)
if not data:
raise FileNotFoundError(rel)
is_dir = bool(data.get("is_dir"))
name = str(data.get("name") or (rel.rstrip("/").split("/")[-1] if rel else ""))
size = int(data.get("size") or 0) if not is_dir else 0
mtime = _parse_iso_to_epoch(data.get("modified"))
info = {
"name": name,
"is_dir": is_dir,
"size": size,
"mtime": mtime,
"type": "dir" if is_dir else "file",
"path": path,
}
return info
async def stat_path(self, root: str, rel: str):
try:
info = await self.stat_file(root, rel)
return {"exists": True, "is_dir": bool(info.get("is_dir")), "path": info.get("path")}
except FileNotFoundError:
return {"exists": False, "is_dir": None, "path": _join_fs_path(root, rel)}
async def exists(self, root: str, rel: str) -> bool:
try:
await self.stat_file(root, rel)
return True
except FileNotFoundError:
return False
except Exception:
return False
async def get_direct_download_response(self, root: str, rel: str):
if not self.enable_redirect_307:
return None
data = await self._fs_get(_join_fs_path(root, rel))
if not data:
raise FileNotFoundError(rel)
if bool(data.get("is_dir")):
raise IsADirectoryError(rel)
raw_url = self._abs_url(str(data.get("raw_url") or ""))
if not raw_url:
return None
return Response(status_code=307, headers={"Location": raw_url})
async def _get_raw_url_and_meta(self, root: str, rel: str) -> Tuple[str, int, str]:
data = await self._fs_get(_join_fs_path(root, rel))
if not data:
raise FileNotFoundError(rel)
if bool(data.get("is_dir")):
raise IsADirectoryError(rel)
raw_url = self._abs_url(str(data.get("raw_url") or ""))
if not raw_url:
raise HTTPException(502, detail=f"{self.product_name} missing raw_url")
size = int(data.get("size") or 0)
name = str(data.get("name") or "")
return raw_url, size, name
async def read_file(self, root: str, rel: str) -> bytes:
raw_url, _, _ = await self._get_raw_url_and_meta(root, rel)
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(raw_url)
resp.raise_for_status()
return resp.content
async def stream_file(self, root: str, rel: str, range_header: str | None):
raw_url, file_size, name = await self._get_raw_url_and_meta(root, rel)
mime, _ = mimetypes.guess_type(name or rel)
content_type = mime or "application/octet-stream"
start = 0
end = max(file_size - 1, 0)
status = 200
headers = {
"Accept-Ranges": "bytes",
"Content-Type": content_type,
}
if file_size >= 0:
headers["Content-Length"] = str(file_size)
if range_header and range_header.startswith("bytes="):
try:
part = range_header.removeprefix("bytes=")
s, e = part.split("-", 1)
if s.strip():
start = int(s)
if e.strip():
end = int(e)
if file_size and start >= file_size:
raise HTTPException(416, detail="Requested Range Not Satisfiable")
if file_size and end >= file_size:
end = file_size - 1
status = 206
except ValueError:
raise HTTPException(400, detail="Invalid Range header")
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
headers["Content-Length"] = str(end - start + 1)
async def agen():
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
req_headers = {"Range": f"bytes={start}-{end}"} if status == 206 else {}
async with client.stream("GET", raw_url, headers=req_headers) as resp:
resp.raise_for_status()
async for chunk in resp.aiter_bytes():
if chunk:
yield chunk
return StreamingResponse(agen(), status_code=status, headers=headers, media_type=content_type)
async def _upload_file(self, full_path: str, file_path: Path) -> Any:
token = await self._ensure_token()
headers = {"File-Path": quote(full_path, safe="/")}
if token:
headers["Authorization"] = token
with file_path.open("rb") as f:
files = {"file": (file_path.name, f, "application/octet-stream")}
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.put(self.base_url + "/api/fs/form", headers=headers, files=files)
resp.raise_for_status()
payload = resp.json()
if not isinstance(payload, dict):
raise HTTPException(502, detail=f"{self.product_name} upload: invalid response")
code = payload.get("code")
if code not in (0, 200):
msg = payload.get("message") or payload.get("msg") or ""
raise HTTPException(502, detail=f"{self.product_name} upload failed: {msg}")
return payload.get("data")
async def write_file(self, root: str, rel: str, data: bytes):
full_path = _join_fs_path(root, rel)
suffix = Path(rel).suffix
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tf:
tf.write(data)
tmp_path = Path(tf.name)
try:
await self._upload_file(full_path, tmp_path)
finally:
try:
tmp_path.unlink(missing_ok=True)
except Exception:
pass
async def write_upload_file(self, root: str, rel: str, file_obj, filename: str | None, file_size: int | None = None, content_type: str | None = None):
full_path = _join_fs_path(root, rel)
token = await self._ensure_token()
headers = {"File-Path": quote(full_path, safe="/")}
if token:
headers["Authorization"] = token
name = filename or Path(rel).name or "file"
mime = content_type or "application/octet-stream"
files = {"file": (name, file_obj, mime)}
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.put(self.base_url + "/api/fs/form", headers=headers, files=files)
resp.raise_for_status()
payload = resp.json()
if not isinstance(payload, dict):
raise HTTPException(502, detail=f"{self.product_name} upload: invalid response")
code = payload.get("code")
if code not in (0, 200):
msg = payload.get("message") or payload.get("msg") or ""
raise HTTPException(502, detail=f"{self.product_name} upload failed: {msg}")
data = payload.get("data")
if isinstance(data, dict) and file_size is not None and "size" not in data:
data["size"] = file_size
return data
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
full_path = _join_fs_path(root, rel)
suffix = Path(rel).suffix
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tf:
tmp_path = Path(tf.name)
size = 0
try:
with tmp_path.open("wb") as f:
async for chunk in data_iter:
if not chunk:
continue
f.write(chunk)
size += len(chunk)
await self._upload_file(full_path, tmp_path)
return size
finally:
try:
tmp_path.unlink(missing_ok=True)
except Exception:
pass
async def mkdir(self, root: str, rel: str):
path = _join_fs_path(root, rel)
await self._api_json("POST", "/api/fs/mkdir", json={"path": path})
async def delete(self, root: str, rel: str):
path = _join_fs_path(root, rel)
parent, name = _split_parent_and_name(path)
if not name:
return
await self._api_json("POST", "/api/fs/remove", json={"dir": parent, "names": [name]})
async def move(self, root: str, src_rel: str, dst_rel: str):
src_path = _join_fs_path(root, src_rel)
dst_path = _join_fs_path(root, dst_rel)
src_dir, src_name = _split_parent_and_name(src_path)
dst_dir, dst_name = _split_parent_and_name(dst_path)
if not src_name or not dst_name:
raise HTTPException(400, detail="Invalid move path")
if src_dir == dst_dir:
if src_name == dst_name:
return
await self._api_json("POST", "/api/fs/rename", json={"path": src_path, "name": dst_name})
return
await self._api_json("POST", "/api/fs/move", json={"src_dir": src_dir, "dst_dir": dst_dir, "names": [src_name]})
if src_name != dst_name:
await self._api_json("POST", "/api/fs/rename", json={"path": _join_fs_path(dst_dir, src_name), "name": dst_name})
async def rename(self, root: str, src_rel: str, dst_rel: str):
await self.move(root, src_rel, dst_rel)
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
src_path = _join_fs_path(root, src_rel)
dst_path = _join_fs_path(root, dst_rel)
src_dir, src_name = _split_parent_and_name(src_path)
dst_dir, dst_name = _split_parent_and_name(dst_path)
if not src_name or not dst_name:
raise HTTPException(400, detail="Invalid copy path")
src_info = await self._fs_get(src_path)
if not src_info:
raise FileNotFoundError(src_rel)
if src_name != dst_name and not bool(src_info.get("is_dir")):
raw_url, _, _ = await self._get_raw_url_and_meta(root, src_rel)
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
async with client.stream("GET", raw_url) as resp:
resp.raise_for_status()
async def gen():
async for chunk in resp.aiter_bytes():
if chunk:
yield chunk
await self.write_file_stream(root, dst_rel, gen())
return
await self._api_json("POST", "/api/fs/copy", json={"src_dir": src_dir, "dst_dir": dst_dir, "names": [src_name]})
if src_name != dst_name:
await self._api_json("POST", "/api/fs/rename", json={"path": _join_fs_path(dst_dir, src_name), "name": dst_name})
class AListAdapter(AListApiAdapterBase):
def __init__(self, record: StorageAdapter):
super().__init__(record, product_name="AList")
class OpenListAdapter(AListApiAdapterBase):
def __init__(self, record: StorageAdapter):
super().__init__(record, product_name="OpenList")
ADAPTER_TYPES = {"alist": AListAdapter, "openlist": OpenListAdapter}
CONFIG_SCHEMA = [
{"key": "base_url", "label": "基础地址", "type": "string", "required": True, "placeholder": "http://127.0.0.1:5244"},
{"key": "username", "label": "用户名", "type": "string", "required": False, "placeholder": "留空则匿名访问"},
{"key": "password", "label": "密码", "type": "password", "required": False, "placeholder": "留空则匿名访问"},
{"key": "root", "label": "根目录", "type": "string", "required": False, "default": "/"},
{"key": "timeout", "label": "超时(秒)", "type": "number", "required": False, "default": 30},
{"key": "enable_direct_download_307", "label": "启用 307 直链下载", "type": "boolean", "default": False},
]
+23
View File
@@ -0,0 +1,23 @@
from typing import List, Dict, Protocol, runtime_checkable, Tuple, AsyncIterator
from models import StorageAdapter
# 约定:任意新适配器模块需定义:
# ADAPTER_TYPE: str
# CONFIG_SCHEMA: List[Dict]
# ADAPTER_FACTORY: Callable[[StorageAdapter], BaseAdapter] (可省略, 会自动寻找 *Adapter 类)
@runtime_checkable
class BaseAdapter(Protocol):
record: StorageAdapter
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]: ...
async def read_file(self, root: str, rel: str) -> bytes: ...
async def write_file(self, root: str, rel: str, data: bytes): ...
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]): ...
async def mkdir(self, root: str, rel: str): ...
async def delete(self, root: str, rel: str): ...
async def move(self, root: str, src_rel: str, dst_rel: str): ...
async def rename(self, root: str, src_rel: str, dst_rel: str): ...
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False): ...
async def stream_file(self, root: str, rel: str, range_header: str | None): ...
async def stat_file(self, root: str, rel: str): ...
def get_effective_root(self, sub_path: str | None) -> str: ...
+471
View File
@@ -0,0 +1,471 @@
import asyncio
import base64
import json
import mimetypes
import re
from datetime import datetime, timezone, timedelta
from typing import AsyncIterator, Dict, List, Tuple
import httpx
from fastapi import HTTPException
from fastapi.responses import Response, StreamingResponse
from models import StorageAdapter
DROPBOX_OAUTH_URL = "https://api.dropboxapi.com/oauth2/token"
DROPBOX_API_URL = "https://api.dropboxapi.com/2"
DROPBOX_CONTENT_URL = "https://content.dropboxapi.com/2"
def _normalize_dbx_path(path: str | None) -> str:
path = (path or "").replace("\\", "/").strip()
if not path or path == "/":
return ""
if not path.startswith("/"):
path = "/" + path
path = re.sub(r"/{2,}", "/", path)
if path.endswith("/"):
path = path.rstrip("/")
return path
def _join_dbx_path(base: str, rel: str) -> str:
base = _normalize_dbx_path(base)
rel = (rel or "").replace("\\", "/").strip("/")
if not rel:
return base
if not base:
return "/" + rel
return f"{base}/{rel}"
def _parse_iso_to_epoch(value: str | None) -> int:
if not value:
return 0
text = str(value).strip()
if not text:
return 0
try:
if text.endswith("Z"):
text = text[:-1] + "+00:00"
dt = datetime.fromisoformat(text)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return int(dt.timestamp())
except Exception:
return 0
class DropboxAdapter:
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config or {}
self.app_key: str = str(cfg.get("app_key") or "").strip()
self.app_secret: str = str(cfg.get("app_secret") or "").strip()
self.refresh_token: str = str(cfg.get("refresh_token") or "").strip()
self.root_path: str = _normalize_dbx_path(str(cfg.get("root") or "/"))
self.enable_redirect_307: bool = bool(cfg.get("enable_direct_download_307"))
self.timeout: float = float(cfg.get("timeout", 60))
if not (self.app_key and self.app_secret and self.refresh_token):
raise ValueError("Dropbox 适配器需要 app_key, app_secret, refresh_token")
self._access_token: str | None = None
self._token_expiry: datetime | None = None
self._token_lock = asyncio.Lock()
def get_effective_root(self, sub_path: str | None) -> str:
base = _normalize_dbx_path(self.root_path)
if sub_path:
return _join_dbx_path(base, sub_path)
return base
async def _get_access_token(self) -> str:
if self._access_token and self._token_expiry and datetime.now(timezone.utc) < self._token_expiry:
return self._access_token
async with self._token_lock:
if self._access_token and self._token_expiry and datetime.now(timezone.utc) < self._token_expiry:
return self._access_token
basic = base64.b64encode(f"{self.app_key}:{self.app_secret}".encode("utf-8")).decode("ascii")
headers = {"Authorization": f"Basic {basic}"}
data = {"grant_type": "refresh_token", "refresh_token": self.refresh_token}
async with httpx.AsyncClient(timeout=self.timeout) as client:
resp = await client.post(DROPBOX_OAUTH_URL, data=data, headers=headers)
resp.raise_for_status()
payload = resp.json()
token = str(payload.get("access_token") or "").strip()
if not token:
raise HTTPException(502, detail="Dropbox oauth: missing access_token")
expires_in = int(payload.get("expires_in") or 3600)
self._access_token = token
self._token_expiry = datetime.now(timezone.utc) + timedelta(seconds=max(60, expires_in - 300))
return token
async def _api_json(self, endpoint: str, body: Dict) -> httpx.Response:
token = await self._get_access_token()
headers = {"Authorization": f"Bearer {token}"}
async with httpx.AsyncClient(timeout=self.timeout) as client:
return await client.post(f"{DROPBOX_API_URL}{endpoint}", json=body, headers=headers)
async def _content_request(
self,
endpoint: str,
api_arg: Dict,
*,
content: bytes | None = None,
data_iter: AsyncIterator[bytes] | None = None,
extra_headers: Dict[str, str] | None = None,
) -> httpx.Response:
token = await self._get_access_token()
headers = {
"Authorization": f"Bearer {token}",
"Dropbox-API-Arg": json.dumps(api_arg, separators=(",", ":"), ensure_ascii=False),
}
if extra_headers:
headers.update(extra_headers)
if data_iter is None:
async with httpx.AsyncClient(timeout=self.timeout) as client:
return await client.post(f"{DROPBOX_CONTENT_URL}{endpoint}", headers=headers, content=content or b"")
async with httpx.AsyncClient(timeout=self.timeout) as client:
return await client.post(f"{DROPBOX_CONTENT_URL}{endpoint}", headers=headers, content=data_iter)
@staticmethod
def _raise_dbx_error(resp: httpx.Response, *, rel: str):
try:
payload = resp.json()
except Exception:
payload = None
summary = ""
if isinstance(payload, dict):
summary = str(payload.get("error_summary") or "")
if "not_found" in summary:
raise FileNotFoundError(rel)
if "conflict" in summary or "already_exists" in summary:
raise FileExistsError(rel)
if "is_folder" in summary:
raise IsADirectoryError(rel)
if "not_folder" in summary:
raise NotADirectoryError(rel)
raise HTTPException(502, detail=f"Dropbox API error: {summary or resp.text}")
def _format_entry(self, entry: Dict) -> Dict:
tag = entry.get(".tag")
is_dir = tag == "folder"
mtime = _parse_iso_to_epoch(entry.get("server_modified") if not is_dir else None)
return {
"name": entry.get("name") or "",
"is_dir": is_dir,
"size": 0 if is_dir else int(entry.get("size") or 0),
"mtime": mtime,
"type": "dir" if is_dir else "file",
}
async def list_dir(
self,
root: str,
rel: str,
page_num: int = 1,
page_size: int = 50,
sort_by: str = "name",
sort_order: str = "asc",
) -> Tuple[List[Dict], int]:
path = _join_dbx_path(root, rel)
body = {"path": path, "recursive": False, "include_deleted": False, "limit": 2000}
resp = await self._api_json("/files/list_folder", body)
if resp.status_code == 409:
try:
payload = resp.json()
except Exception:
payload = None
summary = str((payload or {}).get("error_summary") or "")
if "not_found" in summary:
return [], 0
self._raise_dbx_error(resp, rel=rel)
resp.raise_for_status()
payload = resp.json()
all_entries: List[Dict] = []
all_entries.extend(payload.get("entries") or [])
cursor = payload.get("cursor")
has_more = bool(payload.get("has_more"))
while has_more and cursor:
resp2 = await self._api_json("/files/list_folder/continue", {"cursor": cursor})
resp2.raise_for_status()
p2 = resp2.json()
all_entries.extend(p2.get("entries") or [])
cursor = p2.get("cursor")
has_more = bool(p2.get("has_more"))
items = [self._format_entry(e) for e in all_entries if isinstance(e, dict)]
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
key = (not item["is_dir"],)
f = sort_by.lower()
if f == "name":
key += (item["name"].lower(),)
elif f == "size":
key += (item["size"],)
elif f == "mtime":
key += (item["mtime"],)
else:
key += (item["name"].lower(),)
return key
items.sort(key=get_sort_key, reverse=reverse)
total = len(items)
start = (page_num - 1) * page_size
end = start + page_size
return items[start:end], total
async def stat_file(self, root: str, rel: str):
path = _join_dbx_path(root, rel)
resp = await self._api_json("/files/get_metadata", {"path": path, "include_deleted": False})
if resp.status_code == 409:
self._raise_dbx_error(resp, rel=rel)
resp.raise_for_status()
meta = resp.json()
if not isinstance(meta, dict):
raise HTTPException(502, detail="Dropbox metadata: invalid response")
return self._format_entry(meta)
async def exists(self, root: str, rel: str) -> bool:
try:
await self.stat_file(root, rel)
return True
except FileNotFoundError:
return False
except Exception:
return False
async def read_file(self, root: str, rel: str) -> bytes:
path = _join_dbx_path(root, rel)
resp = await self._content_request("/files/download", {"path": path})
if resp.status_code == 409:
self._raise_dbx_error(resp, rel=rel)
resp.raise_for_status()
return resp.content
async def write_file(self, root: str, rel: str, data: bytes):
path = _join_dbx_path(root, rel)
arg = {
"path": path,
"mode": "overwrite",
"autorename": False,
"mute": False,
"strict_conflict": False,
}
resp = await self._content_request(
"/files/upload",
arg,
content=data,
extra_headers={"Content-Type": "application/octet-stream"},
)
if resp.status_code == 409:
self._raise_dbx_error(resp, rel=rel)
resp.raise_for_status()
return True
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
path = _join_dbx_path(root, rel)
size = 0
session_id: str | None = None
offset = 0
async for chunk in data_iter:
if not chunk:
continue
if session_id is None:
resp = await self._content_request(
"/files/upload_session_start",
{"close": False},
content=chunk,
extra_headers={"Content-Type": "application/octet-stream"},
)
resp.raise_for_status()
payload = resp.json()
session_id = str(payload.get("session_id") or "")
if not session_id:
raise HTTPException(502, detail="Dropbox upload_session_start: missing session_id")
offset += len(chunk)
size += len(chunk)
continue
arg = {"cursor": {"session_id": session_id, "offset": offset}, "close": False}
resp = await self._content_request(
"/files/upload_session_append_v2",
arg,
content=chunk,
extra_headers={"Content-Type": "application/octet-stream"},
)
resp.raise_for_status()
offset += len(chunk)
size += len(chunk)
if session_id is None:
await self.write_file(root, rel, b"")
return 0
finish_arg = {
"cursor": {"session_id": session_id, "offset": offset},
"commit": {
"path": path,
"mode": "overwrite",
"autorename": False,
"mute": False,
"strict_conflict": False,
},
}
resp = await self._content_request(
"/files/upload_session_finish",
finish_arg,
content=b"",
extra_headers={"Content-Type": "application/octet-stream"},
)
if resp.status_code == 409:
self._raise_dbx_error(resp, rel=rel)
resp.raise_for_status()
return size
async def mkdir(self, root: str, rel: str):
path = _join_dbx_path(root, rel)
resp = await self._api_json("/files/create_folder_v2", {"path": path, "autorename": False})
if resp.status_code == 409:
self._raise_dbx_error(resp, rel=rel)
resp.raise_for_status()
return True
async def delete(self, root: str, rel: str):
path = _join_dbx_path(root, rel)
resp = await self._api_json("/files/delete_v2", {"path": path})
if resp.status_code == 409:
try:
payload = resp.json()
except Exception:
payload = None
summary = str((payload or {}).get("error_summary") or "")
if "not_found" in summary:
return
self._raise_dbx_error(resp, rel=rel)
resp.raise_for_status()
return True
async def move(self, root: str, src_rel: str, dst_rel: str):
src = _join_dbx_path(root, src_rel)
dst = _join_dbx_path(root, dst_rel)
resp = await self._api_json(
"/files/move_v2",
{"from_path": src, "to_path": dst, "autorename": False, "allow_shared_folder": True},
)
if resp.status_code == 409:
self._raise_dbx_error(resp, rel=src_rel)
resp.raise_for_status()
return True
async def rename(self, root: str, src_rel: str, dst_rel: str):
return await self.move(root, src_rel, dst_rel)
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
src = _join_dbx_path(root, src_rel)
dst = _join_dbx_path(root, dst_rel)
resp = await self._api_json(
"/files/copy_v2",
{"from_path": src, "to_path": dst, "autorename": False, "allow_shared_folder": True},
)
if resp.status_code == 409:
self._raise_dbx_error(resp, rel=dst_rel if overwrite else dst_rel)
resp.raise_for_status()
return True
async def get_direct_download_response(self, root: str, rel: str):
if not self.enable_redirect_307:
return None
path = _join_dbx_path(root, rel)
resp = await self._api_json("/files/get_temporary_link", {"path": path})
if resp.status_code == 409:
self._raise_dbx_error(resp, rel=rel)
resp.raise_for_status()
payload = resp.json()
link = (payload.get("link") if isinstance(payload, dict) else None) or ""
link = str(link).strip()
if not link:
return None
return Response(status_code=307, headers={"Location": link})
async def stream_file(self, root: str, rel: str, range_header: str | None):
path = _join_dbx_path(root, rel)
token = await self._get_access_token()
headers = {
"Authorization": f"Bearer {token}",
"Dropbox-API-Arg": json.dumps({"path": path}, separators=(",", ":"), ensure_ascii=False),
}
if range_header:
headers["Range"] = range_header
client = httpx.AsyncClient(timeout=None)
stream_cm = client.stream("POST", f"{DROPBOX_CONTENT_URL}/files/download", headers=headers)
try:
resp = await stream_cm.__aenter__()
except Exception:
await client.aclose()
raise
if resp.status_code == 409:
try:
content = await resp.aread()
_ = content
finally:
await stream_cm.__aexit__(None, None, None)
await client.aclose()
self._raise_dbx_error(resp, rel=rel)
if resp.status_code >= 400:
try:
await resp.aread()
finally:
await stream_cm.__aexit__(None, None, None)
await client.aclose()
resp.raise_for_status()
content_type = resp.headers.get("Content-Type") or (mimetypes.guess_type(rel)[0] or "application/octet-stream")
out_headers = {}
for key in ("Accept-Ranges", "Content-Range", "Content-Length"):
value = resp.headers.get(key)
if value:
out_headers[key] = value
async def iterator():
try:
async for chunk in resp.aiter_bytes():
if chunk:
yield chunk
finally:
await stream_cm.__aexit__(None, None, None)
await client.aclose()
return StreamingResponse(iterator(), status_code=resp.status_code, headers=out_headers, media_type=content_type)
ADAPTER_TYPE = "dropbox"
CONFIG_SCHEMA = [
{"key": "app_key", "label": "App Key", "type": "string", "required": True},
{"key": "app_secret", "label": "App Secret", "type": "password", "required": True},
{"key": "refresh_token", "label": "Refresh Token", "type": "password", "required": True},
{"key": "root", "label": "Root Path", "type": "string", "required": False, "default": "/", "placeholder": "/ or /Apps/Foxel"},
{"key": "timeout", "label": "超时(秒)", "type": "number", "required": False, "default": 60},
{"key": "enable_direct_download_307", "label": "Enable 307 redirect download", "type": "boolean", "default": False},
]
def ADAPTER_FACTORY(rec): return DropboxAdapter(rec)
+435
View File
@@ -0,0 +1,435 @@
import asyncio
import mimetypes
import re
import tempfile
from pathlib import Path
from typing import Any, AsyncIterator, Dict, List, Tuple
from urllib.parse import quote
import httpx
from fastapi import HTTPException
from fastapi.responses import StreamingResponse
from models import StorageAdapter
def _normalize_fs_path(path: str) -> str:
path = (path or "").replace("\\", "/").strip()
if not path or path == "/":
return "/"
if not path.startswith("/"):
path = "/" + path
path = re.sub(r"/{2,}", "/", path)
if path != "/" and path.endswith("/"):
path = path.rstrip("/")
return path or "/"
def _join_fs_path(base: str, rel: str | None) -> str:
base = _normalize_fs_path(base)
rel_norm = (rel or "").replace("\\", "/").strip().lstrip("/")
if not rel_norm:
return base
if base == "/":
return "/" + rel_norm
return f"{base}/{rel_norm}"
def _unwrap_success(payload: Any, *, context: str) -> Any:
if not isinstance(payload, dict):
return payload
if "data" not in payload:
return payload
code = payload.get("code")
if code not in (None, 0, 200):
msg = payload.get("msg") or payload.get("message") or ""
raise HTTPException(502, detail=f"Foxel 上游错误({context}): {msg}")
return payload.get("data")
class FoxelAdapter:
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config or {}
self.base_url: str = str(cfg.get("base_url", "")).rstrip("/")
if not self.base_url.startswith("http"):
raise ValueError("foxel requires base_url http/https")
self.username: str = str(cfg.get("username") or "")
self.password: str = str(cfg.get("password") or "")
if not self.username or not self.password:
raise ValueError("foxel requires username and password")
self.timeout: float = float(cfg.get("timeout", 15))
self.root_path: str = _normalize_fs_path(str(cfg.get("root") or "/"))
self._token: str | None = None
self._login_lock = asyncio.Lock()
def get_effective_root(self, sub_path: str | None) -> str:
base = _normalize_fs_path(self.root_path)
if sub_path:
return _join_fs_path(base, sub_path)
return base
async def _login(self) -> str:
url = self.base_url + "/api/auth/login"
body = {"username": self.username, "password": self.password}
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.post(url, data=body)
resp.raise_for_status()
payload = resp.json()
if not isinstance(payload, dict):
raise HTTPException(502, detail="Foxel 登录响应异常")
token = payload.get("access_token")
if not token:
raise HTTPException(502, detail="Foxel 登录失败: 缺少 access_token")
return str(token)
async def _ensure_token(self) -> str:
if self._token:
return self._token
async with self._login_lock:
if self._token:
return self._token
self._token = await self._login()
return self._token
async def _request_json(self, method: str, path: str, *, params: dict | None = None, json: Any = None) -> Any:
url = self.base_url + path
for attempt in range(2):
token = await self._ensure_token()
headers = {"Authorization": f"Bearer {token}"}
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.request(method, url, headers=headers, params=params, json=json)
if resp.status_code == 401 and attempt == 0:
self._token = None
continue
resp.raise_for_status()
return resp.json()
raise HTTPException(502, detail="Foxel 上游请求失败")
@staticmethod
def _encode_path(full_path: str) -> str:
return quote(full_path.lstrip("/"), safe="/")
def _browse_path(self, full_path: str) -> str:
full_path = _normalize_fs_path(full_path)
if full_path == "/":
return "/api/fs/"
return "/api/fs/" + self._encode_path(full_path)
def _stat_path(self, full_path: str) -> str:
full_path = _normalize_fs_path(full_path)
if full_path == "/":
return "/api/fs/stat/"
return "/api/fs/stat/" + self._encode_path(full_path)
def _file_path(self, full_path: str) -> str:
full_path = _normalize_fs_path(full_path)
if full_path == "/":
return "/api/fs/file/"
return "/api/fs/file/" + self._encode_path(full_path)
def _stream_path(self, full_path: str) -> str:
full_path = _normalize_fs_path(full_path)
if full_path == "/":
return "/api/fs/stream/"
return "/api/fs/stream/" + self._encode_path(full_path)
async def list_dir(
self,
root: str,
rel: str,
page_num: int = 1,
page_size: int = 50,
sort_by: str = "name",
sort_order: str = "asc",
) -> Tuple[List[Dict], int]:
rel = (rel or "").strip("/")
full_path = _join_fs_path(root, rel)
payload = await self._request_json(
"GET",
self._browse_path(full_path),
params={
"page": page_num,
"page_size": page_size,
"sort_by": sort_by,
"sort_order": sort_order,
},
)
data = _unwrap_success(payload, context="list_dir")
if not isinstance(data, dict):
raise HTTPException(502, detail="Foxel 浏览响应异常")
entries = data.get("entries") or []
pagination = data.get("pagination") or {}
total = pagination.get("total")
try:
total_int = int(total) if total is not None else len(entries)
except Exception:
total_int = len(entries)
if not isinstance(entries, list):
entries = []
return entries, total_int
async def stat_file(self, root: str, rel: str):
rel = (rel or "").strip("/")
full_path = _join_fs_path(root, rel)
payload = await self._request_json("GET", self._stat_path(full_path))
data = _unwrap_success(payload, context="stat_file")
if not isinstance(data, dict):
raise HTTPException(502, detail="Foxel stat 响应异常")
return data
async def exists(self, root: str, rel: str) -> bool:
rel = (rel or "").strip("/")
full_path = _join_fs_path(root, rel)
url = self.base_url + self._stat_path(full_path)
for attempt in range(2):
token = await self._ensure_token()
headers = {"Authorization": f"Bearer {token}"}
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url, headers=headers)
if resp.status_code == 401 and attempt == 0:
self._token = None
continue
return resp.status_code == 200
return False
async def read_file(self, root: str, rel: str) -> bytes:
rel = (rel or "").lstrip("/")
full_path = _join_fs_path(root, rel)
url = self.base_url + self._file_path(full_path)
for attempt in range(2):
token = await self._ensure_token()
headers = {"Authorization": f"Bearer {token}"}
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.get(url, headers=headers)
if resp.status_code == 401 and attempt == 0:
self._token = None
continue
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
return resp.content
raise HTTPException(502, detail="Foxel 读取失败")
async def _upload_file_path(self, full_path: str, file_path: Path) -> None:
url = self.base_url + self._file_path(full_path)
filename = Path(full_path).name or file_path.name
for attempt in range(2):
token = await self._ensure_token()
headers = {"Authorization": f"Bearer {token}"}
with file_path.open("rb") as f:
files = {"file": (filename, f, "application/octet-stream")}
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.post(url, headers=headers, files=files)
if resp.status_code == 401 and attempt == 0:
self._token = None
continue
resp.raise_for_status()
return
raise HTTPException(502, detail="Foxel 上传失败")
async def write_file(self, root: str, rel: str, data: bytes):
rel = (rel or "").lstrip("/")
full_path = _join_fs_path(root, rel)
url = self.base_url + self._file_path(full_path)
filename = Path(rel).name or "file"
for attempt in range(2):
token = await self._ensure_token()
headers = {"Authorization": f"Bearer {token}"}
files = {"file": (filename, data, "application/octet-stream")}
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.post(url, headers=headers, files=files)
if resp.status_code == 401 and attempt == 0:
self._token = None
continue
resp.raise_for_status()
return True
raise HTTPException(502, detail="Foxel 写入失败")
async def write_upload_file(self, root: str, rel: str, file_obj, filename: str | None, file_size: int | None = None, content_type: str | None = None):
rel = (rel or "").lstrip("/")
full_path = _join_fs_path(root, rel)
url = self.base_url + self._file_path(full_path)
name = filename or Path(rel).name or "file"
mime = content_type or "application/octet-stream"
for attempt in range(2):
try:
if callable(getattr(file_obj, "seek", None)):
file_obj.seek(0)
except Exception:
pass
token = await self._ensure_token()
headers = {"Authorization": f"Bearer {token}"}
files = {"file": (name, file_obj, mime)}
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.post(url, headers=headers, files=files)
if resp.status_code == 401 and attempt == 0:
self._token = None
continue
resp.raise_for_status()
return {"size": file_size or 0}
raise HTTPException(502, detail="Foxel 上传失败")
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
rel = (rel or "").lstrip("/")
full_path = _join_fs_path(root, rel)
suffix = Path(rel).suffix
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tf:
tmp_path = Path(tf.name)
size = 0
try:
with tmp_path.open("wb") as f:
async for chunk in data_iter:
if not chunk:
continue
f.write(chunk)
size += len(chunk)
await self._upload_file_path(full_path, tmp_path)
return size
finally:
try:
tmp_path.unlink(missing_ok=True)
except Exception:
pass
async def mkdir(self, root: str, rel: str):
rel = (rel or "").strip("/")
full_path = _join_fs_path(root, rel)
payload = await self._request_json("POST", "/api/fs/mkdir", json={"path": full_path})
_unwrap_success(payload, context="mkdir")
return True
async def delete(self, root: str, rel: str):
rel = (rel or "").strip("/")
full_path = _join_fs_path(root, rel)
url = self.base_url + self._browse_path(full_path)
for attempt in range(2):
token = await self._ensure_token()
headers = {"Authorization": f"Bearer {token}"}
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
resp = await client.delete(url, headers=headers)
if resp.status_code == 401 and attempt == 0:
self._token = None
continue
if resp.status_code == 404:
return
resp.raise_for_status()
return
raise HTTPException(502, detail="Foxel 删除失败")
async def move(self, root: str, src_rel: str, dst_rel: str):
src_path = _join_fs_path(root, (src_rel or "").lstrip("/"))
dst_path = _join_fs_path(root, (dst_rel or "").lstrip("/"))
payload = await self._request_json("POST", "/api/fs/move", json={"src": src_path, "dst": dst_path})
_unwrap_success(payload, context="move")
return True
async def rename(self, root: str, src_rel: str, dst_rel: str):
src_path = _join_fs_path(root, (src_rel or "").lstrip("/"))
dst_path = _join_fs_path(root, (dst_rel or "").lstrip("/"))
payload = await self._request_json("POST", "/api/fs/rename", json={"src": src_path, "dst": dst_path})
_unwrap_success(payload, context="rename")
return True
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
src_path = _join_fs_path(root, (src_rel or "").lstrip("/"))
dst_path = _join_fs_path(root, (dst_rel or "").lstrip("/"))
payload = await self._request_json(
"POST",
"/api/fs/copy",
json={"src": src_path, "dst": dst_path},
params={"overwrite": overwrite},
)
_unwrap_success(payload, context="copy")
return True
async def stream_file(self, root: str, rel: str, range_header: str | None):
rel = (rel or "").lstrip("/")
full_path = _join_fs_path(root, rel)
url = self.base_url + self._stream_path(full_path)
headers = {}
if range_header:
headers["Range"] = range_header
for attempt in range(2):
token = await self._ensure_token()
headers["Authorization"] = f"Bearer {token}"
client = httpx.AsyncClient(timeout=None, follow_redirects=True)
stream_cm = client.stream("GET", url, headers=headers)
try:
resp = await stream_cm.__aenter__()
except Exception:
await client.aclose()
raise
if resp.status_code == 401 and attempt == 0:
try:
await resp.aread()
finally:
await stream_cm.__aexit__(None, None, None)
await client.aclose()
self._token = None
continue
if resp.status_code == 404:
try:
await resp.aread()
finally:
await stream_cm.__aexit__(None, None, None)
await client.aclose()
raise FileNotFoundError(rel)
if resp.status_code >= 400:
try:
await resp.aread()
finally:
await stream_cm.__aexit__(None, None, None)
await client.aclose()
resp.raise_for_status()
content_type = resp.headers.get("Content-Type") or (
mimetypes.guess_type(rel)[0] or "application/octet-stream"
)
out_headers = {}
for key in ("Accept-Ranges", "Content-Range", "Content-Length"):
value = resp.headers.get(key)
if value:
out_headers[key] = value
async def iterator():
try:
async for chunk in resp.aiter_bytes():
if chunk:
yield chunk
finally:
await stream_cm.__aexit__(None, None, None)
await client.aclose()
return StreamingResponse(
iterator(),
status_code=resp.status_code,
headers=out_headers,
media_type=content_type,
)
raise HTTPException(502, detail="Foxel 流式读取失败")
ADAPTER_TYPE = "foxel"
CONFIG_SCHEMA = [
{"key": "base_url", "label": "节点地址", "type": "string", "required": True, "placeholder": "http://127.0.0.1:8000"},
{"key": "username", "label": "用户名", "type": "string", "required": True},
{"key": "password", "label": "密码", "type": "password", "required": True},
{"key": "root", "label": "远端根目录", "type": "string", "required": False, "default": "/", "placeholder": "/ 或 /drive"},
{"key": "timeout", "label": "超时(秒)", "type": "number", "required": False, "default": 60},
]
def ADAPTER_FACTORY(rec: StorageAdapter):
return FoxelAdapter(rec)
+645
View File
@@ -0,0 +1,645 @@
import asyncio
from dataclasses import dataclass
from typing import List, Dict, Tuple, AsyncIterator, Optional
from fastapi import HTTPException
from fastapi.responses import StreamingResponse
from ftplib import FTP, error_perm
import mimetypes
from models import StorageAdapter
def _join_remote(root: str, rel: str) -> str:
root = (root or "/").rstrip("/") or "/"
rel = (rel or "").lstrip("/")
if not rel:
return root
return f"{root}/{rel}"
def _parse_mlst_line(line: str) -> Dict[str, str]:
out: Dict[str, str] = {}
try:
facts, _, name = line.partition(" ")
for part in facts.split(";"):
if not part or "=" not in part:
continue
k, v = part.split("=", 1)
out[k.strip().lower()] = v.strip()
if name:
out["name"] = name.strip()
except Exception:
pass
return out
def _parse_modify_to_epoch(mod: str) -> int:
# Formats we may see: YYYYMMDDHHMMSS or YYYYMMDDHHMMSS(.sss)
try:
mod = mod.strip()
mod = mod.split(".")[0]
if len(mod) >= 14:
y = int(mod[0:4])
m = int(mod[4:6])
d = int(mod[6:8])
hh = int(mod[8:10])
mm = int(mod[10:12])
ss = int(mod[12:14])
import datetime as _dt
return int(_dt.datetime(y, m, d, hh, mm, ss, tzinfo=_dt.timezone.utc).timestamp())
except Exception:
return 0
return 0
@dataclass
class _Range:
start: int
end: Optional[int] # inclusive
class FTPAdapter:
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config
self.host: str = cfg.get("host")
self.port: int = int(cfg.get("port", 21))
self.username: Optional[str] = cfg.get("username")
self.password: Optional[str] = cfg.get("password")
self.passive: bool = bool(cfg.get("passive", True))
self.timeout: int = int(cfg.get("timeout", 15))
self.root_path: str = cfg.get("root", "/") or "/"
if not self.host:
raise ValueError("FTP adapter requires 'host'")
def get_effective_root(self, sub_path: str | None) -> str:
base = self.root_path.rstrip("/") or "/"
if sub_path:
return _join_remote(base, sub_path)
return base
def _connect(self) -> FTP:
ftp = FTP()
ftp.connect(self.host, self.port, timeout=self.timeout)
if self.username:
ftp.login(self.username, self.password or "")
else:
ftp.login()
ftp.set_pasv(self.passive)
return ftp
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
path = _join_remote(root, rel.strip('/'))
def _do_list() -> List[Dict]:
ftp = self._connect()
try:
ftp.cwd(path)
except error_perm as e:
# path may be file
ftp.quit()
raise NotADirectoryError(rel) from e
entries: List[Dict] = []
# Try MLSD first
try:
for name, facts in ftp.mlsd():
if name in (".", ".."):
continue
is_dir = (facts.get("type") == "dir")
size = int(facts.get("size") or 0)
mtime = _parse_modify_to_epoch(facts.get("modify") or "")
entries.append({
"name": name,
"is_dir": is_dir,
"size": 0 if is_dir else size,
"mtime": mtime,
"type": "dir" if is_dir else "file",
})
ftp.quit()
return entries
except Exception:
# Fallback to NLST + probing
pass
names = []
try:
names = ftp.nlst()
except Exception:
ftp.quit()
return []
for name in names:
if name in (".", ".."):
continue
is_dir = False
size = 0
mtime = 0
try:
# If we can CWD, it's a directory
ftp.cwd(_join_remote(path, name))
ftp.cwd(path)
is_dir = True
except Exception:
is_dir = False
try:
size = ftp.size(_join_remote(path, name)) or 0
except Exception:
size = 0
try:
mdtm = ftp.sendcmd("MDTM " + _join_remote(path, name))
# Example: '213 20241012XXXXXX'
if mdtm.startswith("213 "):
mtime = _parse_modify_to_epoch(mdtm.split(" ", 1)[1])
except Exception:
pass
entries.append({
"name": name,
"is_dir": is_dir,
"size": 0 if is_dir else int(size or 0),
"mtime": int(mtime or 0),
"type": "dir" if is_dir else "file",
})
ftp.quit()
return entries
entries = await asyncio.to_thread(_do_list)
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
key = (not item["is_dir"],)
f = sort_by.lower()
if f == "name":
key += (item["name"].lower(),)
elif f == "size":
key += (item.get("size", 0),)
elif f == "mtime":
key += (item.get("mtime", 0),)
else:
key += (item["name"].lower(),)
return key
entries.sort(key=get_sort_key, reverse=reverse)
total = len(entries)
start = (page_num - 1) * page_size
end = start + page_size
return entries[start:end], total
async def read_file(self, root: str, rel: str) -> bytes:
path = _join_remote(root, rel)
def _do_read() -> bytes:
ftp = self._connect()
try:
chunks: List[bytes] = []
ftp.retrbinary("RETR " + path, lambda b: chunks.append(b))
return b"".join(chunks)
except error_perm as e:
if str(e).startswith("550"):
raise FileNotFoundError(rel)
raise
finally:
try:
ftp.quit()
except Exception:
pass
return await asyncio.to_thread(_do_read)
async def write_file(self, root: str, rel: str, data: bytes):
path = _join_remote(root, rel)
def _ensure_dirs(ftp: FTP, dir_path: str):
parts = [p for p in dir_path.strip("/").split("/") if p]
cur = "/"
for p in parts:
cur = _join_remote(cur, p)
try:
ftp.mkd(cur)
except Exception:
pass
def _do_write():
ftp = self._connect()
try:
parent = "/" if "/" not in path.strip("/") else path.rsplit("/", 1)[0]
_ensure_dirs(ftp, parent)
from io import BytesIO
bio = BytesIO(data)
ftp.storbinary("STOR " + path, bio)
finally:
try:
ftp.quit()
except Exception:
pass
await asyncio.to_thread(_do_write)
async def write_upload_file(self, root: str, rel: str, file_obj, filename: str | None, file_size: int | None = None, content_type: str | None = None):
path = _join_remote(root, rel)
def _ensure_dirs(ftp: FTP, dir_path: str):
parts = [p for p in dir_path.strip("/").split("/") if p]
cur = "/"
for p in parts:
cur = _join_remote(cur, p)
try:
ftp.mkd(cur)
except Exception:
pass
def _do_upload():
ftp = self._connect()
try:
parent = "/" if "/" not in path.strip("/") else path.rsplit("/", 1)[0]
_ensure_dirs(ftp, parent)
try:
if callable(getattr(file_obj, "seek", None)):
file_obj.seek(0)
except Exception:
pass
ftp.storbinary("STOR " + path, file_obj)
finally:
try:
ftp.quit()
except Exception:
pass
await asyncio.to_thread(_do_upload)
return {"size": file_size or 0}
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
# KISS: 聚合后一次性写入
buf = bytearray()
async for chunk in data_iter:
if chunk:
buf.extend(chunk)
await self.write_file(root, rel, bytes(buf))
return len(buf)
async def mkdir(self, root: str, rel: str):
path = _join_remote(root, rel)
def _do_mkdir():
ftp = self._connect()
try:
parts = [p for p in path.strip("/").split("/") if p]
cur = "/"
for p in parts:
cur = _join_remote(cur, p)
try:
ftp.mkd(cur)
except Exception:
pass
finally:
try:
ftp.quit()
except Exception:
pass
await asyncio.to_thread(_do_mkdir)
async def delete(self, root: str, rel: str):
path = _join_remote(root, rel)
def _do_delete():
ftp = self._connect()
try:
# Try file delete
try:
ftp.delete(path)
return
except Exception:
pass
# Recursively delete dir
def _rm_tree(dir_path: str):
try:
ftp.cwd(dir_path)
except Exception:
return
items = []
try:
for name, facts in ftp.mlsd():
if name in (".", ".."):
continue
items.append((name, facts.get("type") == "dir"))
except Exception:
try:
names = ftp.nlst()
except Exception:
names = []
for n in names:
if n in (".", ".."):
continue
# Best-effort dir check
try:
ftp.cwd(_join_remote(dir_path, n))
ftp.cwd(dir_path)
items.append((n, True))
except Exception:
items.append((n, False))
for n, is_dir in items:
child = _join_remote(dir_path, n)
if is_dir:
_rm_tree(child)
else:
try:
ftp.delete(child)
except Exception:
pass
try:
ftp.rmd(dir_path)
except Exception:
pass
_rm_tree(path)
finally:
try:
ftp.quit()
except Exception:
pass
await asyncio.to_thread(_do_delete)
async def move(self, root: str, src_rel: str, dst_rel: str):
src = _join_remote(root, src_rel)
dst = _join_remote(root, dst_rel)
def _do_move():
ftp = self._connect()
try:
# Ensure dst parent exists
parent = "/" if "/" not in dst.strip("/") else dst.rsplit("/", 1)[0]
parts = [p for p in parent.strip("/").split("/") if p]
cur = "/"
for p in parts:
cur = _join_remote(cur, p)
try:
ftp.mkd(cur)
except Exception:
pass
ftp.rename(src, dst)
finally:
try:
ftp.quit()
except Exception:
pass
await asyncio.to_thread(_do_move)
async def rename(self, root: str, src_rel: str, dst_rel: str):
await self.move(root, src_rel, dst_rel)
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
src = _join_remote(root, src_rel)
dst = _join_remote(root, dst_rel)
# naive implementation: download then upload; recursively for dirs
async def _is_dir(path: str) -> bool:
def _probe() -> bool:
ftp = self._connect()
try:
try:
ftp.cwd(path)
return True
except Exception:
return False
finally:
try:
ftp.quit()
except Exception:
pass
return await asyncio.to_thread(_probe)
if await _is_dir(src):
# list children, create dst dir, copy recursively
await self.mkdir(root, dst_rel)
children, _ = await self.list_dir(root, src_rel, page_num=1, page_size=10_000)
for ent in children:
child_src = f"{src_rel.rstrip('/')}/{ent['name']}"
child_dst = f"{dst_rel.rstrip('/')}/{ent['name']}"
await self.copy(root, child_src, child_dst, overwrite)
return
# file
data = await self.read_file(root, src_rel)
if not overwrite:
# best-effort existence check
try:
await self.stat_file(root, dst_rel)
raise FileExistsError(dst_rel)
except FileNotFoundError:
pass
await self.write_file(root, dst_rel, data)
async def stat_file(self, root: str, rel: str):
path = _join_remote(root, rel)
def _do_stat():
ftp = self._connect()
try:
# Try MLST
try:
resp: List[str] = []
ftp.retrlines("MLST " + path, resp.append)
# The last line usually contains facts
facts = {}
if resp:
facts = _parse_mlst_line(resp[-1])
name = rel.split("/")[-1]
t = facts.get("type") or "file"
is_dir = t == "dir"
size = int(facts.get("size") or 0)
mtime = _parse_modify_to_epoch(facts.get("modify") or "")
return {
"name": name,
"is_dir": is_dir,
"size": 0 if is_dir else size,
"mtime": mtime,
"type": "dir" if is_dir else "file",
"path": path,
}
except Exception:
pass
# Probe directory
try:
ftp.cwd(path)
return {
"name": rel.split("/")[-1],
"is_dir": True,
"size": 0,
"mtime": 0,
"type": "dir",
"path": path,
}
except Exception:
pass
# Treat as file
try:
size = ftp.size(path) or 0
except Exception:
size = 0
try:
mdtm = ftp.sendcmd("MDTM " + path)
mtime = _parse_modify_to_epoch(mdtm.split(" ", 1)[1]) if mdtm.startswith("213 ") else 0
except Exception:
mtime = 0
return {
"name": rel.split("/")[-1],
"is_dir": False,
"size": int(size or 0),
"mtime": int(mtime or 0),
"type": "file",
"path": path,
}
except error_perm as e:
if str(e).startswith("550"):
raise FileNotFoundError(rel)
raise
finally:
try:
ftp.quit()
except Exception:
pass
return await asyncio.to_thread(_do_stat)
async def stream_file(self, root: str, rel: str, range_header: str | None):
path = _join_remote(root, rel)
# Get size (best-effort)
def _get_size() -> Optional[int]:
ftp = self._connect()
try:
try:
return int(ftp.size(path) or 0)
except Exception:
return None
finally:
try:
ftp.quit()
except Exception:
pass
total_size = await asyncio.to_thread(_get_size)
mime, _ = mimetypes.guess_type(rel)
content_type = mime or "application/octet-stream"
rng: Optional[_Range] = None
status = 200
headers = {"Accept-Ranges": "bytes", "Content-Type": content_type}
if range_header and range_header.startswith("bytes=") and total_size is not None:
try:
s, e = (range_header.removeprefix("bytes=").split("-", 1))
start = int(s) if s.strip() else 0
end = int(e) if e.strip() else (total_size - 1)
if start >= total_size:
raise HTTPException(416, detail="Requested Range Not Satisfiable")
if end >= total_size:
end = total_size - 1
rng = _Range(start, end)
status = 206
headers["Content-Range"] = f"bytes {start}-{end}/{total_size}"
headers["Content-Length"] = str(end - start + 1)
except ValueError:
raise HTTPException(400, detail="Invalid Range header")
elif total_size is not None:
headers["Content-Length"] = str(total_size)
queue: asyncio.Queue[Optional[bytes]] = asyncio.Queue(maxsize=8)
class _Stop(Exception):
pass
def _worker():
ftp = self._connect()
remaining = None
if rng is not None:
remaining = (rng.end - rng.start + 1) if rng.end is not None else None
def _cb(chunk: bytes):
nonlocal remaining
if not chunk:
return
try:
if remaining is not None:
if len(chunk) > remaining:
part = chunk[:remaining]
queue.put_nowait(part)
remaining = 0
raise _Stop()
else:
queue.put_nowait(chunk)
remaining -= len(chunk)
if remaining <= 0:
raise _Stop()
else:
queue.put_nowait(chunk)
except _Stop:
raise
except Exception:
# queue full or event loop closed
raise _Stop()
try:
if rng is not None:
ftp.retrbinary("RETR " + path, _cb, rest=rng.start)
else:
ftp.retrbinary("RETR " + path, _cb)
queue.put_nowait(None)
except _Stop:
try:
queue.put_nowait(None)
except Exception:
pass
except error_perm as e:
try:
queue.put_nowait(None)
except Exception:
pass
if str(e).startswith("550"):
pass
finally:
try:
ftp.quit()
except Exception:
pass
async def agen():
worker_fut = asyncio.to_thread(_worker)
try:
while True:
chunk = await queue.get()
if chunk is None:
break
yield chunk
finally:
try:
await worker_fut
except Exception:
pass
return StreamingResponse(agen(), status_code=status, headers=headers, media_type=content_type)
ADAPTER_TYPE = "ftp"
CONFIG_SCHEMA = [
{"key": "host", "label": "主机", "type": "string", "required": True, "placeholder": "ftp.example.com"},
{"key": "port", "label": "端口", "type": "number", "required": False, "default": 21},
{"key": "username", "label": "用户名", "type": "string", "required": False},
{"key": "password", "label": "密码", "type": "password", "required": False},
{"key": "passive", "label": "被动模式", "type": "boolean", "required": False, "default": True},
{"key": "timeout", "label": "超时(秒)", "type": "number", "required": False, "default": 15},
{"key": "root", "label": "根路径", "type": "string", "required": False, "default": "/"},
]
def ADAPTER_FACTORY(rec: StorageAdapter):
return FTPAdapter(rec)
+559
View File
@@ -0,0 +1,559 @@
from datetime import datetime, timezone, timedelta
from typing import List, Dict, Tuple, AsyncIterator
import httpx
from fastapi.responses import StreamingResponse, Response
from fastapi import HTTPException
from models import StorageAdapter
GOOGLE_OAUTH_URL = "https://oauth2.googleapis.com/token"
GOOGLE_DRIVE_API_URL = "https://www.googleapis.com/drive/v3"
class GoogleDriveAdapter:
"""Google Drive 存储适配器"""
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config
self.client_id = cfg.get("client_id")
self.client_secret = cfg.get("client_secret")
self.refresh_token = cfg.get("refresh_token")
self.root_folder_id = cfg.get("root_folder_id", "root")
self.enable_redirect_307 = bool(cfg.get("enable_direct_download_307"))
if not all([self.client_id, self.client_secret, self.refresh_token]):
raise ValueError(
"Google Drive 适配器需要 client_id, client_secret, 和 refresh_token")
self._access_token: str | None = None
self._token_expiry: datetime | None = None
def get_effective_root(self, sub_path: str | None) -> str:
"""
获取有效根路径。
:param sub_path: 子路径。
:return: 完整的有效路径。
"""
if sub_path:
return f"{sub_path.strip('/')}".strip()
return ""
async def _get_access_token(self) -> str:
"""
获取或刷新 access token。
:return: access token。
"""
if self._access_token and self._token_expiry and datetime.now(timezone.utc) < self._token_expiry:
return self._access_token
data = {
"client_id": self.client_id,
"client_secret": self.client_secret,
"refresh_token": self.refresh_token,
"grant_type": "refresh_token",
}
async with httpx.AsyncClient(timeout=20.0) as client:
resp = await client.post(GOOGLE_OAUTH_URL, data=data)
resp.raise_for_status()
token_data = resp.json()
self._access_token = token_data["access_token"]
self._token_expiry = datetime.now(
timezone.utc) + timedelta(seconds=token_data["expires_in"] - 300)
return self._access_token
async def _request(self, method: str, endpoint: str, **kwargs):
"""
向 Google Drive API 发送请求。
:param method: HTTP 方法。
:param endpoint: API 端点。
:param kwargs: 其他请求参数。
:return: 响应对象。
"""
token = await self._get_access_token()
headers = {"Authorization": f"Bearer {token}"}
if "headers" in kwargs:
headers.update(kwargs.pop("headers"))
url = f"{GOOGLE_DRIVE_API_URL}{endpoint}"
async with httpx.AsyncClient(timeout=60.0) as client:
resp = await client.request(method, url, headers=headers, **kwargs)
if resp.status_code == 401:
self._access_token = None
token = await self._get_access_token()
headers["Authorization"] = f"Bearer {token}"
resp = await client.request(method, url, headers=headers, **kwargs)
return resp
async def _get_folder_id_by_path(self, path: str) -> str:
"""
通过路径获取文件夹 ID。
:param path: 路径。
:return: 文件夹 ID。
"""
if not path or path == "/":
return self.root_folder_id
parts = [p for p in path.strip("/").split("/") if p]
current_id = self.root_folder_id
for part in parts:
query = f"name='{part}' and '{current_id}' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false"
params = {"q": query, "fields": "files(id, name)"}
resp = await self._request("GET", "/files", params=params)
resp.raise_for_status()
data = resp.json()
files = data.get("files", [])
if not files:
raise FileNotFoundError(f"文件夹不存在: {part}")
current_id = files[0]["id"]
return current_id
async def _get_file_id_by_path(self, path: str) -> str | None:
"""
通过路径获取文件 ID。
:param path: 文件路径。
:return: 文件 ID 或 None。
"""
if not path or path == "/":
return self.root_folder_id
parts = [p for p in path.strip("/").split("/") if p]
parent_id = self.root_folder_id
for i, part in enumerate(parts):
is_last = i == len(parts) - 1
mime_filter = "" if is_last else "and mimeType='application/vnd.google-apps.folder'"
query = f"name='{part}' and '{parent_id}' in parents {mime_filter} and trashed=false"
params = {"q": query, "fields": "files(id, name)"}
resp = await self._request("GET", "/files", params=params)
resp.raise_for_status()
data = resp.json()
files = data.get("files", [])
if not files:
return None
parent_id = files[0]["id"]
return parent_id
def _format_item(self, item: Dict) -> Dict:
"""
将 Google Drive API 返回的 item 格式化为统一的格式。
:param item: Google Drive API 返回的 item 字典。
:return: 格式化后的字典。
"""
is_dir = item["mimeType"] == "application/vnd.google-apps.folder"
mtime_str = item.get("modifiedTime", item.get("createdTime", ""))
try:
mtime = int(datetime.fromisoformat(mtime_str.replace("Z", "+00:00")).timestamp())
except:
mtime = 0
return {
"name": item["name"],
"is_dir": is_dir,
"size": 0 if is_dir else int(item.get("size", 0)),
"mtime": mtime,
"type": "dir" if is_dir else "file",
}
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
"""
列出目录内容。
:param root: 根路径。
:param rel: 相对路径。
:param page_num: 页码。
:param page_size: 每页大小。
:param sort_by: 排序字段
:param sort_order: 排序顺序
:return: 文件/目录列表和总数。
"""
try:
folder_id = await self._get_folder_id_by_path(rel)
except FileNotFoundError:
return [], 0
query = f"'{folder_id}' in parents and trashed=false"
params = {
"q": query,
"fields": "files(id, name, mimeType, size, modifiedTime, createdTime)",
"pageSize": 1000,
}
all_items = []
page_token = None
while True:
if page_token:
params["pageToken"] = page_token
resp = await self._request("GET", "/files", params=params)
if resp.status_code == 404:
return [], 0
resp.raise_for_status()
data = resp.json()
all_items.extend(data.get("files", []))
page_token = data.get("nextPageToken")
if not page_token:
break
formatted_items = [self._format_item(item) for item in all_items]
# 排序
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
key = (not item["is_dir"],)
sort_field = sort_by.lower()
if sort_field == "name":
key += (item["name"].lower(),)
elif sort_field == "size":
key += (item["size"],)
elif sort_field == "mtime":
key += (item["mtime"],)
else:
key += (item["name"].lower(),)
return key
formatted_items.sort(key=get_sort_key, reverse=reverse)
total_count = len(formatted_items)
start_idx = (page_num - 1) * page_size
end_idx = start_idx + page_size
return formatted_items[start_idx:end_idx], total_count
async def read_file(self, root: str, rel: str) -> bytes:
"""
读取文件内容。
:param root: 根路径。
:param rel: 相对路径。
:return: 文件内容的字节流。
"""
file_id = await self._get_file_id_by_path(rel)
if not file_id:
raise FileNotFoundError(rel)
resp = await self._request("GET", f"/files/{file_id}", params={"alt": "media"})
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
return resp.content
async def write_file(self, root: str, rel: str, data: bytes):
"""
写入文件。
:param root: 根路径。
:param rel: 相对路径。
:param data: 文件内容的字节流。
"""
parent_path = "/".join(rel.strip("/").split("/")[:-1])
file_name = rel.strip("/").split("/")[-1]
parent_id = await self._get_folder_id_by_path(parent_path)
# 检查文件是否已存在
existing_id = await self._get_file_id_by_path(rel)
if existing_id:
# 更新现有文件
async with httpx.AsyncClient(timeout=60.0) as client:
token = await self._get_access_token()
headers = {"Authorization": f"Bearer {token}"}
url = f"https://www.googleapis.com/upload/drive/v3/files/{existing_id}?uploadType=media"
resp = await client.patch(url, headers=headers, content=data)
resp.raise_for_status()
else:
# 创建新文件
metadata = {
"name": file_name,
"parents": [parent_id]
}
async with httpx.AsyncClient(timeout=60.0) as client:
token = await self._get_access_token()
headers = {"Authorization": f"Bearer {token}"}
# 使用 multipart 上传
import json
boundary = "===============boundary==============="
headers["Content-Type"] = f"multipart/related; boundary={boundary}"
body = (
f"--{boundary}\r\n"
f"Content-Type: application/json; charset=UTF-8\r\n\r\n"
f"{json.dumps(metadata)}\r\n"
f"--{boundary}\r\n"
f"Content-Type: application/octet-stream\r\n\r\n"
).encode() + data + f"\r\n--{boundary}--".encode()
url = "https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart"
resp = await client.post(url, headers=headers, content=body)
resp.raise_for_status()
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
"""
以流式方式写入文件。
:param root: 根路径。
:param rel: 相对路径。
:param data_iter: 文件内容的异步迭代器。
:return: 文件大小。
"""
# 先收集所有数据
chunks = []
total_size = 0
async for chunk in data_iter:
chunks.append(chunk)
total_size += len(chunk)
data = b"".join(chunks)
await self.write_file(root, rel, data)
return total_size
async def mkdir(self, root: str, rel: str):
"""
创建目录。
:param root: 根路径。
:param rel: 相对路径。
"""
parent_path = "/".join(rel.strip("/").split("/")[:-1])
folder_name = rel.strip("/").split("/")[-1]
parent_id = await self._get_folder_id_by_path(parent_path)
metadata = {
"name": folder_name,
"mimeType": "application/vnd.google-apps.folder",
"parents": [parent_id]
}
resp = await self._request("POST", "/files", json=metadata)
resp.raise_for_status()
async def delete(self, root: str, rel: str):
"""
删除文件或目录。
:param root: 根路径。
:param rel: 相对路径。
"""
file_id = await self._get_file_id_by_path(rel)
if not file_id:
return
resp = await self._request("DELETE", f"/files/{file_id}")
if resp.status_code not in (204, 404):
resp.raise_for_status()
async def move(self, root: str, src_rel: str, dst_rel: str):
"""
移动或重命名文件/目录。
:param root: 根路径。
:param src_rel: 源相对路径。
:param dst_rel: 目标相对路径。
"""
file_id = await self._get_file_id_by_path(src_rel)
if not file_id:
raise FileNotFoundError(src_rel)
# 获取当前父文件夹
resp = await self._request("GET", f"/files/{file_id}", params={"fields": "parents"})
resp.raise_for_status()
current_parents = resp.json().get("parents", [])
# 获取目标父文件夹和新名称
dst_parent_path = "/".join(dst_rel.strip("/").split("/")[:-1])
dst_name = dst_rel.strip("/").split("/")[-1]
dst_parent_id = await self._get_folder_id_by_path(dst_parent_path)
# 更新文件
params = {
"addParents": dst_parent_id,
"removeParents": ",".join(current_parents) if current_parents else None,
}
metadata = {"name": dst_name}
resp = await self._request("PATCH", f"/files/{file_id}", params=params, json=metadata)
resp.raise_for_status()
async def rename(self, root: str, src_rel: str, dst_rel: str):
"""
重命名文件或目录。
"""
await self.move(root, src_rel, dst_rel)
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
"""
复制文件或目录。
:param root: 根路径。
:param src_rel: 源相对路径。
:param dst_rel: 目标相对路径。
:param overwrite: 是否覆盖。
"""
file_id = await self._get_file_id_by_path(src_rel)
if not file_id:
raise FileNotFoundError(src_rel)
dst_parent_path = "/".join(dst_rel.strip("/").split("/")[:-1])
dst_name = dst_rel.strip("/").split("/")[-1]
dst_parent_id = await self._get_folder_id_by_path(dst_parent_path)
metadata = {
"name": dst_name,
"parents": [dst_parent_id]
}
resp = await self._request("POST", f"/files/{file_id}/copy", json=metadata)
resp.raise_for_status()
async def stream_file(self, root: str, rel: str, range_header: str | None):
"""
流式传输文件(支持范围请求)。
:param root: 根路径。
:param rel: 相对路径。
:param range_header: HTTP Range 头。
:return: FastAPI StreamingResponse 对象。
"""
file_id = await self._get_file_id_by_path(rel)
if not file_id:
raise FileNotFoundError(rel)
# 获取文件元数据
resp = await self._request("GET", f"/files/{file_id}", params={"fields": "name, size, mimeType"})
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
item_data = resp.json()
file_size = int(item_data.get("size", 0))
content_type = item_data.get("mimeType", "application/octet-stream")
start = 0
end = file_size - 1
status = 200
headers = {
"Accept-Ranges": "bytes",
"Content-Type": content_type,
"Content-Disposition": f"inline; filename=\"{item_data.get('name')}\""
}
if range_header and range_header.startswith("bytes="):
try:
part = range_header.removeprefix("bytes=")
s, e = part.split("-", 1)
if s.strip():
start = int(s)
if e.strip():
end = int(e)
if start >= file_size:
raise HTTPException(416, "Requested Range Not Satisfiable")
if end >= file_size:
end = file_size - 1
status = 206
except ValueError:
raise HTTPException(400, "Invalid Range header")
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
headers["Content-Length"] = str(end - start + 1)
else:
headers["Content-Length"] = str(file_size)
async def file_iterator():
nonlocal start, end
token = await self._get_access_token()
async with httpx.AsyncClient(timeout=60.0) as client:
req_headers = {
'Authorization': f'Bearer {token}',
'Range': f'bytes={start}-{end}'
}
url = f"{GOOGLE_DRIVE_API_URL}/files/{file_id}?alt=media"
async with client.stream("GET", url, headers=req_headers) as stream_resp:
stream_resp.raise_for_status()
async for chunk in stream_resp.aiter_bytes():
yield chunk
return StreamingResponse(file_iterator(), status_code=status, headers=headers, media_type=content_type)
async def stat_file(self, root: str, rel: str):
"""
获取文件或目录的元数据。
:param root: 根路径。
:param rel: 相对路径。
:return: 格式化后的文件/目录信息。
"""
file_id = await self._get_file_id_by_path(rel)
if not file_id:
raise FileNotFoundError(rel)
resp = await self._request("GET", f"/files/{file_id}", params={"fields": "id, name, mimeType, size, modifiedTime, createdTime"})
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
return self._format_item(resp.json())
async def get_direct_download_response(self, root: str, rel: str):
"""
获取直接下载响应 (307 重定向)。
:param root: 根路径。
:param rel: 相对路径。
:return: 307 重定向响应或 None。
"""
if not self.enable_redirect_307:
return None
file_id = await self._get_file_id_by_path(rel)
if not file_id:
raise FileNotFoundError(rel)
# 获取文件的下载链接
resp = await self._request("GET", f"/files/{file_id}", params={"fields": "webContentLink"})
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
item_data = resp.json()
download_url = item_data.get("webContentLink")
if not download_url:
return None
return Response(status_code=307, headers={"Location": download_url})
async def get_thumbnail(self, root: str, rel: str, size: str = "medium"):
"""
获取文件的缩略图。
:param root: 根路径。
:param rel: 相对路径。
:param size: 缩略图大小 (暂未使用,Google Drive 自动决定)。
:return: 缩略图内容的字节流,或在不支持时返回 None。
"""
file_id = await self._get_file_id_by_path(rel)
if not file_id:
return None
try:
resp = await self._request("GET", f"/files/{file_id}", params={"fields": "thumbnailLink"})
if resp.status_code == 200:
item_data = resp.json()
thumbnail_link = item_data.get("thumbnailLink")
if thumbnail_link:
async with httpx.AsyncClient(timeout=30.0) as client:
thumb_resp = await client.get(thumbnail_link)
thumb_resp.raise_for_status()
return thumb_resp.content
return None
except Exception:
return None
ADAPTER_TYPE = "googledrive"
CONFIG_SCHEMA = [
{"key": "client_id", "label": "Client ID", "type": "string", "required": True},
{"key": "client_secret", "label": "Client Secret",
"type": "password", "required": True},
{"key": "refresh_token", "label": "Refresh Token", "type": "password",
"required": True, "help_text": "可以通过 Google OAuth 2.0 Playground 获取"},
{"key": "root_folder_id", "label": "根文件夹 ID (Root Folder ID)", "type": "string",
"required": False, "placeholder": "默认为根目录 (root)", "default": "root"},
{"key": "enable_direct_download_307", "label": "Enable 307 redirect download", "type": "boolean", "default": False},
]
def ADAPTER_FACTORY(rec): return GoogleDriveAdapter(rec)
+337
View File
@@ -0,0 +1,337 @@
import os
import shutil
import stat
import time
from pathlib import Path
from typing import List, Dict, Tuple, AsyncIterator
import asyncio
import mimetypes
from fastapi import HTTPException
from fastapi.responses import StreamingResponse, Response
from models import StorageAdapter
def _safe_join(root: str, rel: str) -> Path:
root_path = Path(root).resolve()
full = (root_path / rel).resolve()
if not str(full).startswith(str(root_path)):
raise ValueError("Path escape detected")
return full
DEFAULT_FILE_MODE = 0o666
DEFAULT_DIR_MODE = 0o777
def _apply_mode(path: Path, mode: int):
try:
os.chmod(path, mode)
except Exception:
pass
class LocalAdapter:
def __init__(self, record: StorageAdapter):
self.record = record
self.root = self.record.config.get("root")
if not self.root:
raise ValueError("Local adapter config requires 'root'")
Path(self.root).mkdir(parents=True, exist_ok=True)
def get_effective_root(self, sub_path: str | None) -> str:
root = self.record.config.get("root")
if sub_path:
return str(Path(root) / sub_path)
return root
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
rel = rel.strip('/')
base = _safe_join(root, rel) if rel else Path(root)
if not base.exists():
return [], 0
if not base.is_dir():
raise NotADirectoryError(rel)
all_names = await asyncio.to_thread(os.listdir, base)
entries = []
for name in all_names:
fp = base / name
try:
st = await asyncio.to_thread(fp.stat)
except FileNotFoundError:
continue
is_dir = fp.is_dir()
entries.append({
"name": name,
"is_dir": is_dir,
"size": 0 if is_dir else st.st_size,
"mtime": int(st.st_mtime),
"mode": stat.S_IMODE(st.st_mode),
"type": "dir" if is_dir else "file",
})
# 排序
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
# 基础排序键,目录优先
key = (not item["is_dir"],)
sort_field = sort_by.lower()
if sort_field == "name":
key += (item["name"].lower(),)
elif sort_field == "size":
key += (item["size"],)
elif sort_field == "mtime":
key += (item["mtime"],)
else: # 默认按名称
key += (item["name"].lower(),)
return key
entries.sort(key=get_sort_key, reverse=reverse)
total_count = len(entries)
# 分页
start_idx = (page_num - 1) * page_size
end_idx = start_idx + page_size
page_entries = entries[start_idx:end_idx]
return page_entries, total_count
async def read_file(self, root: str, rel: str) -> bytes:
fp = _safe_join(root, rel)
if not fp.exists() or not fp.is_file():
raise FileNotFoundError(rel)
return await asyncio.to_thread(fp.read_bytes)
async def write_file(self, root: str, rel: str, data: bytes):
fp = _safe_join(root, rel)
pre_exists = fp.exists()
await asyncio.to_thread(os.makedirs, fp.parent, mode=DEFAULT_DIR_MODE, exist_ok=True)
await asyncio.to_thread(fp.write_bytes, data)
if not pre_exists:
await asyncio.to_thread(_apply_mode, fp, DEFAULT_FILE_MODE)
async def write_upload_file(self, root: str, rel: str, file_obj, filename: str | None, file_size: int | None = None, content_type: str | None = None):
fp = _safe_join(root, rel)
pre_exists = fp.exists()
await asyncio.to_thread(os.makedirs, fp.parent, mode=DEFAULT_DIR_MODE, exist_ok=True)
def _copy():
try:
if callable(getattr(file_obj, "seek", None)):
file_obj.seek(0)
except Exception:
pass
with open(fp, "wb") as f:
shutil.copyfileobj(file_obj, f)
await asyncio.to_thread(_copy)
if not pre_exists:
await asyncio.to_thread(_apply_mode, fp, DEFAULT_FILE_MODE)
size = file_size
if size is None:
try:
size = fp.stat().st_size
except Exception:
size = 0
return {"size": int(size or 0)}
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
fp = _safe_join(root, rel)
pre_exists = fp.exists()
await asyncio.to_thread(os.makedirs, fp.parent, mode=DEFAULT_DIR_MODE, exist_ok=True)
# 流式写入,避免一次性读入内存
def _open():
return open(fp, "wb")
f = await asyncio.to_thread(_open)
size = 0
try:
async for chunk in data_iter:
if not chunk:
continue
size += len(chunk)
await asyncio.to_thread(f.write, chunk)
finally:
await asyncio.to_thread(f.close)
if not pre_exists:
await asyncio.to_thread(_apply_mode, fp, DEFAULT_FILE_MODE)
return size
async def mkdir(self, root: str, rel: str):
fp = _safe_join(root, rel)
await asyncio.to_thread(os.makedirs, fp, mode=DEFAULT_DIR_MODE, exist_ok=True)
async def delete(self, root: str, rel: str):
fp = _safe_join(root, rel)
if not fp.exists():
return
if fp.is_dir():
await asyncio.to_thread(shutil.rmtree, fp)
else:
await asyncio.to_thread(fp.unlink)
async def stat_path(self, root: str, rel: str):
"""新增: 返回路径状态调试信息"""
fp = _safe_join(root, rel)
def _stat():
if not fp.exists():
return {"exists": False, "is_dir": None, "path": str(fp)}
return {
"exists": True,
"is_dir": fp.is_dir(),
"path": str(fp)
}
return await asyncio.to_thread(_stat)
async def exists(self, root: str, rel: str) -> bool:
"""新增: 判断路径是否存在"""
fp = _safe_join(root, rel)
return await asyncio.to_thread(fp.exists)
async def move(self, root: str, src_rel: str, dst_rel: str):
src = _safe_join(root, src_rel)
dst = _safe_join(root, dst_rel)
if str(src) == str(dst):
return
if not src.exists():
raise FileNotFoundError(src_rel)
await asyncio.to_thread(dst.parent.mkdir, parents=True, exist_ok=True)
def _do_move():
try:
os.replace(src, dst)
except OSError:
shutil.move(str(src), str(dst))
await asyncio.to_thread(_do_move)
async def rename(self, root: str, src_rel: str, dst_rel: str):
src = _safe_join(root, src_rel)
dst = _safe_join(root, dst_rel)
if str(src) == str(dst):
return
if not src.exists():
raise FileNotFoundError(src_rel)
await asyncio.to_thread(dst.parent.mkdir, parents=True, exist_ok=True)
def _do_rename():
try:
os.rename(src, dst)
except OSError:
os.replace(src, dst)
await asyncio.to_thread(_do_rename)
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
src = _safe_join(root, src_rel)
dst = _safe_join(root, dst_rel)
if not src.exists():
raise FileNotFoundError(src_rel)
if str(src) == str(dst):
return
await asyncio.to_thread(dst.parent.mkdir, parents=True, exist_ok=True)
def _do():
if dst.exists():
if not overwrite:
raise FileExistsError(dst_rel)
if dst.is_dir():
shutil.rmtree(dst)
else:
dst.unlink()
if src.is_dir():
shutil.copytree(src, dst)
else:
shutil.copy2(src, dst)
await asyncio.to_thread(_do)
async def stream_file(self, root: str, rel: str, range_header: str | None):
fp = _safe_join(root, rel)
if not fp.exists() or not fp.is_file():
raise HTTPException(404, detail="File not found")
mime, _ = mimetypes.guess_type(rel)
content_type = mime or "application/octet-stream"
file_size = (await asyncio.to_thread(fp.stat)).st_size
start = 0
end = file_size - 1
status = 200
headers = {
"Accept-Ranges": "bytes",
"Content-Type": content_type,
}
if range_header and range_header.startswith("bytes="):
try:
part = range_header.removeprefix("bytes=")
s, e = part.split("-", 1)
if s.strip():
start = int(s)
if e.strip():
end = int(e)
if start >= file_size:
raise HTTPException(416, detail="Requested Range Not Satisfiable")
if end >= file_size:
end = file_size - 1
status = 206
except ValueError:
raise HTTPException(400, detail="Invalid Range header")
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
headers["Content-Length"] = str(end - start + 1)
else:
headers["Content-Length"] = str(file_size)
async def iterator():
# 使用线程池避免阻塞
def _read_segment(offset: int, length: int):
with open(fp, "rb") as f:
f.seek(offset)
return f.read(length)
chunk_size = 256 * 1024
remaining = end - start + 1
offset = start
while remaining > 0:
size = min(chunk_size, remaining)
data = await asyncio.to_thread(_read_segment, offset, size)
if not data:
break
yield data
remaining -= len(data)
offset += len(data)
return StreamingResponse(iterator(), status_code=status, headers=headers, media_type=content_type)
async def stat_file(self, root: str, rel: str):
fp = _safe_join(root, rel)
if not fp.exists():
raise FileNotFoundError(rel)
st = await asyncio.to_thread(fp.stat)
info = {
"name": fp.name,
"is_dir": fp.is_dir(),
"size": st.st_size,
"mtime": int(st.st_mtime),
"mode": stat.S_IMODE(st.st_mode),
"type": "dir" if fp.is_dir() else "file",
"path": str(fp),
}
# exif信息
exif = None
if not fp.is_dir():
mime, _ = mimetypes.guess_type(fp.name)
if mime and mime.startswith("image/"):
try:
from PIL import Image
img = await asyncio.to_thread(Image.open, fp)
exif_data = img._getexif()
if exif_data:
exif = {str(k): str(v) for k, v in exif_data.items()}
except Exception:
exif = None
info["exif"] = exif
return info
ADAPTER_TYPE = "local"
CONFIG_SCHEMA = [
{"key": "root", "label": "根目录", "type": "string", "required": True, "placeholder": "/data/storage"},
]
ADAPTER_FACTORY = lambda rec: LocalAdapter(rec)
+461
View File
@@ -0,0 +1,461 @@
from datetime import datetime, timezone, timedelta
from typing import List, Dict, Tuple, AsyncIterator
import httpx
from fastapi.responses import StreamingResponse, Response
from fastapi import HTTPException
from models import StorageAdapter
MS_GRAPH_URL = "https://graph.microsoft.com/v1.0"
MS_OAUTH_URL = "https://login.microsoftonline.com/common/oauth2/v2.0/token"
class OneDriveAdapter:
"""OneDrive 存储适配器"""
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config
self.client_id = cfg.get("client_id")
self.client_secret = cfg.get("client_secret")
self.refresh_token = cfg.get("refresh_token")
self.root = cfg.get("root", "/").strip("/")
self.enable_redirect_307 = bool(cfg.get("enable_direct_download_307"))
if not all([self.client_id, self.client_secret, self.refresh_token]):
raise ValueError(
"OneDrive 适配器需要 client_id, client_secret, 和 refresh_token")
self._access_token: str | None = None
self._token_expiry: datetime | None = None
def get_effective_root(self, sub_path: str | None) -> str:
"""
获取有效根路径。
:param sub_path: 子路径。
:return: 完整的有效路径。
"""
if sub_path:
return f"/{self.root.strip('/')}/{sub_path.strip('/')}".strip()
return f"/{self.root.strip('/')}".strip()
def _get_api_path(self, rel_path: str) -> str:
"""
将用户可见的相对路径转换为 Graph API 路径段。
:param rel_path: 相对路径。
:return: Graph API 路径段。
"""
full_path = self.get_effective_root(rel_path).strip('/')
if not full_path:
return ""
return f":/{full_path}"
async def _get_access_token(self) -> str:
"""
获取或刷新 access token。
:return: access token。
"""
if self._access_token and self._token_expiry and datetime.now(timezone.utc) < self._token_expiry:
return self._access_token
data = {
"client_id": self.client_id,
"client_secret": self.client_secret,
"refresh_token": self.refresh_token,
"grant_type": "refresh_token",
}
async with httpx.AsyncClient(timeout=20.0) as client:
resp = await client.post(MS_OAUTH_URL, data=data)
resp.raise_for_status()
token_data = resp.json()
self._access_token = token_data["access_token"]
self._token_expiry = datetime.now(
timezone.utc) + timedelta(seconds=token_data["expires_in"] - 300)
return self._access_token
async def _request(self, method: str, api_path_segment: str | None = None, *, full_url: str | None = None, **kwargs):
"""
向 Microsoft Graph API 发送请求。
:param method: HTTP 方法。
:param api_path_segment: API 路径段 (与 full_url 互斥)。
:param full_url: 完整的请求 URL (与 api_path_segment 互斥)。
:param kwargs: 其他请求参数。
:return: 响应对象。
"""
if not ((api_path_segment is not None) ^ (full_url is not None)):
raise ValueError("必须提供 api_path_segment 或 full_url 中的一个,且仅一个")
token = await self._get_access_token()
headers = {"Authorization": f"Bearer {token}"}
if "headers" in kwargs:
headers.update(kwargs.pop("headers"))
url = full_url if full_url else f"{MS_GRAPH_URL}/me/drive/root{api_path_segment}"
async with httpx.AsyncClient(timeout=60.0) as client:
resp = await client.request(method, url, headers=headers, **kwargs)
if resp.status_code == 401:
self._access_token = None
token = await self._get_access_token()
headers["Authorization"] = f"Bearer {token}"
resp = await client.request(method, url, headers=headers, **kwargs)
return resp
def _format_item(self, item: Dict) -> Dict:
"""
将 Graph API 返回的 item 格式化为统一的格式。
:param item: Graph API 返回的 item 字典。
:return: 格式化后的字典。
"""
is_dir = "folder" in item
return {
"name": item["name"],
"is_dir": is_dir,
"size": 0 if is_dir else item.get("size", 0),
"mtime": int(datetime.fromisoformat(item["lastModifiedDateTime"].replace("Z", "+00:00")).timestamp()),
"type": "dir" if is_dir else "file",
}
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
"""
列出目录内容。
由于 Graph API 不支持基于偏移($skip)的分页,此方法将获取所有项目,
:param root: 根路径 (在此适配器中未使用,通过配置的 root 确定)。
:param rel: 相对路径。
:param page_num: 页码。
:param page_size: 每页大小。
:param sort_by: 排序字段
:param sort_order: 排序顺序
:return: 文件/目录列表和总数。
"""
api_path = self._get_api_path(rel)
children_path = f"{api_path}:/children" if api_path else "/children"
all_items = []
params = {"$top": 999}
resp = await self._request("GET", api_path_segment=children_path, params=params)
while True:
if resp.status_code == 404 and not all_items:
return [], 0
resp.raise_for_status()
try:
data = resp.json()
except Exception as e:
raise IOError(f"解析 Graph API 响应失败: {e}") from e
all_items.extend(data.get("value", []))
next_link = data.get("@odata.nextLink")
if not next_link:
break
resp = await self._request("GET", full_url=next_link)
formatted_items = [self._format_item(item) for item in all_items]
# 排序
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
key = (not item["is_dir"],)
sort_field = sort_by.lower()
if sort_field == "name":
key += (item["name"].lower(),)
elif sort_field == "size":
key += (item["size"],)
elif sort_field == "mtime":
key += (item["mtime"],)
else:
key += (item["name"].lower(),)
return key
formatted_items.sort(key=get_sort_key, reverse=reverse)
total_count = len(formatted_items)
start_idx = (page_num - 1) * page_size
end_idx = start_idx + page_size
return formatted_items[start_idx:end_idx], total_count
async def read_file(self, root: str, rel: str) -> bytes:
"""
读取文件内容。
:param root: 根路径。
:param rel: 相对路径。
:return: 文件内容的字节流。
"""
api_path = self._get_api_path(rel)
if not api_path:
raise IsADirectoryError("不能将根目录作为文件读取")
resp = await self._request("GET", api_path_segment=f"{api_path}:/content")
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
return resp.content
async def write_file(self, root: str, rel: str, data: bytes):
"""
写入文件。
:param root: 根路径。
:param rel: 相对路径。
:param data: 文件内容的字节流。
"""
api_path = self._get_api_path(rel)
if not api_path:
raise ValueError("不能直接写入根路径")
resp = await self._request("PUT", api_path_segment=f"{api_path}:/content", content=data)
resp.raise_for_status()
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
"""
以流式方式写入文件。
:param root: 根路径。
:param rel: 相对路径。
:param data_iter: 文件内容的异步迭代器。
:return: 文件大小。
"""
api_path = self._get_api_path(rel)
if not api_path:
raise ValueError("不能直接写入根路径")
resp = await self._request("PUT", api_path_segment=f"{api_path}:/content", content=data_iter)
resp.raise_for_status()
return resp.json().get("size", 0)
async def mkdir(self, root: str, rel: str):
"""
创建目录。
:param root: 根路径。
:param rel: 相对路径。
"""
parent_path_str, new_dir_name = rel.rstrip(
'/').rsplit('/', 1) if '/' in rel.rstrip('/') else ('', rel)
parent_api_path = self._get_api_path(parent_path_str)
children_path = f"{parent_api_path}:/children" if parent_api_path else "/children"
payload = {
"name": new_dir_name,
"folder": {},
"@microsoft.graph.conflictBehavior": "fail" # 如果已存在则失败
}
resp = await self._request("POST", api_path_segment=children_path, json=payload)
resp.raise_for_status()
async def delete(self, root: str, rel: str):
"""
删除文件或目录。
:param root: 根路径。
:param rel: 相对路径。
"""
api_path = self._get_api_path(rel)
if not api_path:
raise ValueError("不能删除根目录")
resp = await self._request("DELETE", api_path_segment=api_path)
if resp.status_code not in (204, 404):
resp.raise_for_status()
async def move(self, root: str, src_rel: str, dst_rel: str):
"""
移动或重命名文件/目录。
:param root: 根路径。
:param src_rel: 源相对路径。
:param dst_rel: 目标相对路径。
"""
src_api_path = self._get_api_path(src_rel)
if not src_api_path:
raise ValueError("不能移动根目录")
dst_parent_rel, dst_name = dst_rel.rstrip(
'/').rsplit('/', 1) if '/' in dst_rel.rstrip('/') else ('', dst_rel)
dst_parent_api_path = self._get_api_path(dst_parent_rel)
# 获取父项目的 ID
parent_resp = await self._request("GET", api_path_segment=dst_parent_api_path)
parent_resp.raise_for_status()
parent_id = parent_resp.json()["id"]
payload = {
"parentReference": {"id": parent_id},
"name": dst_name
}
resp = await self._request("PATCH", api_path_segment=src_api_path, json=payload)
resp.raise_for_status()
async def rename(self, root: str, src_rel: str, dst_rel: str):
"""
重命名文件或目录。
在 Graph API 中,移动和重命名是同一个 PATCH 操作。
"""
await self.move(root, src_rel, dst_rel)
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
"""
复制文件或目录。
:param root: 根路径。
:param src_rel: 源相对路径。
:param dst_rel: 目标相对路径。
:param overwrite: 是否覆盖 (在此 API 中未直接使用)。
"""
src_api_path = self._get_api_path(src_rel)
if not src_api_path:
raise ValueError("不能复制根目录")
dst_parent_rel, dst_name = dst_rel.rstrip(
'/').rsplit('/', 1) if '/' in dst_rel.rstrip('/') else ('', dst_rel)
dst_parent_api_path = self._get_api_path(dst_parent_rel)
parent_resp = await self._request("GET", api_path_segment=dst_parent_api_path)
parent_resp.raise_for_status()
parent_id = parent_resp.json()["id"]
payload = {"parentReference": {"id": parent_id}, "name": dst_name}
copy_path = f"{src_api_path}:/copy"
resp = await self._request("POST", api_path_segment=copy_path, json=payload)
resp.raise_for_status()
async def stream_file(self, root: str, rel: str, range_header: str | None):
"""
流式传输文件(支持范围请求)。
:param root: 根路径。
:param rel: 相对路径。
:param range_header: HTTP Range 头。
:return: FastAPI StreamingResponse 对象。
"""
api_path = self._get_api_path(rel)
if not api_path:
raise IsADirectoryError("不能对目录进行流式传输")
resp = await self._request("GET", api_path_segment=api_path)
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
item_data = resp.json()
download_url = item_data.get("@microsoft.graph.downloadUrl")
if not download_url:
raise Exception("无法获取下载 URL")
file_size = item_data.get("size", 0)
content_type = item_data.get("file", {}).get(
"mimeType", "application/octet-stream")
start = 0
end = file_size - 1
status = 200
headers = {
"Accept-Ranges": "bytes",
"Content-Type": content_type,
"Content-Disposition": f"inline; filename=\"{item_data.get('name')}\""
}
if range_header and range_header.startswith("bytes="):
try:
part = range_header.removeprefix("bytes=")
s, e = part.split("-", 1)
if s.strip():
start = int(s)
if e.strip():
end = int(e)
if start >= file_size:
raise HTTPException(416, "Requested Range Not Satisfiable")
if end >= file_size:
end = file_size - 1
status = 206
except ValueError:
raise HTTPException(400, "Invalid Range header")
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
headers["Content-Length"] = str(end - start + 1)
else:
headers["Content-Length"] = str(file_size)
async def file_iterator():
nonlocal start, end
async with httpx.AsyncClient(timeout=60.0) as client:
req_headers = {'Range': f'bytes={start}-{end}'}
async with client.stream("GET", download_url, headers=req_headers) as stream_resp:
stream_resp.raise_for_status()
async for chunk in stream_resp.aiter_bytes():
yield chunk
return StreamingResponse(file_iterator(), status_code=status, headers=headers, media_type=content_type)
async def get_direct_download_response(self, root: str, rel: str):
if not self.enable_redirect_307:
return None
api_path = self._get_api_path(rel)
if not api_path:
raise IsADirectoryError("不能对目录进行直链重定向")
resp = await self._request("GET", api_path_segment=api_path)
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
item_data = resp.json()
download_url = item_data.get("@microsoft.graph.downloadUrl")
if not download_url:
return None
return Response(status_code=307, headers={"Location": download_url})
async def get_thumbnail(self, root: str, rel: str, size: str = "medium"):
"""
获取文件的缩略图。
:param root: 根路径。
:param rel: 相对路径。
:param size: 缩略图大小 (large, medium, small)。
:return: 缩略图内容的字节流,或在不支持时返回 None。
"""
api_path = self._get_api_path(rel)
if not api_path:
return None
thumb_path = f"{api_path}:/thumbnails/0/{size}"
try:
resp = await self._request("GET", api_path_segment=thumb_path)
if resp.status_code == 200:
thumb_data = resp.json()
async with httpx.AsyncClient(timeout=30.0) as client:
thumb_resp = await client.get(thumb_data['url'])
thumb_resp.raise_for_status()
return thumb_resp.content
elif resp.status_code == 404:
return None
else:
resp.raise_for_status()
except Exception:
return None
async def stat_file(self, root: str, rel: str):
"""
获取文件或目录的元数据。
:param root: 根路径。
:param rel: 相对路径。
:return: 格式化后的文件/目录信息。
"""
api_path = self._get_api_path(rel)
resp = await self._request("GET", api_path_segment=api_path)
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
return self._format_item(resp.json())
ADAPTER_TYPE = "onedrive"
CONFIG_SCHEMA = [
{"key": "client_id", "label": "Client ID", "type": "string", "required": True},
{"key": "client_secret", "label": "Client Secret",
"type": "password", "required": True},
{"key": "refresh_token", "label": "Refresh Token", "type": "password",
"required": True, "help_text": "可以通过运行 'python -m domain.adapters.providers.onedrive' 获取"},
{"key": "root", "label": "根目录 (Root Path)", "type": "string",
"required": False, "placeholder": "默认为根目录 /"},
{"key": "enable_direct_download_307", "label": "Enable 307 redirect download", "type": "boolean", "default": False},
]
def ADAPTER_FACTORY(rec): return OneDriveAdapter(rec)
+911
View File
@@ -0,0 +1,911 @@
import asyncio
import base64
import hashlib
import mimetypes
import os
import time
from typing import Dict, List, Tuple, Optional, AsyncIterator, Any
import httpx
from fastapi import HTTPException
from fastapi.responses import StreamingResponse
from models import StorageAdapter
from .base import BaseAdapter
# Quark 普通(UC)接口
API_BASE = "https://drive.quark.cn/1/clouddrive"
REFERER = "https://pan.quark.cn"
PR = "ucpro"
class QuarkAdapter:
"""夸克网盘(Cookie 模式)
- 使用浏览器导出的 Cookie 进行鉴权
- 通过 Quark/UC 的 clouddrive 接口实现:列目录、读写、分片上传、基础操作
- 根 FID 固定为 "0";路径解析通过名称遍历
"""
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config or {}
self.cookie: str = cfg.get("cookie") or cfg.get("Cookie")
self.root_fid: str = cfg.get("root_fid", "0")
def _as_bool(value: Any) -> bool:
if isinstance(value, bool):
return value
if isinstance(value, str):
return value.strip().lower() in {"1", "true", "yes", "on"}
return bool(value)
self.use_transcoding_address: bool = _as_bool(cfg.get("use_transcoding_address", False))
self.only_list_video_file: bool = _as_bool(cfg.get("only_list_video_file", False))
if not self.cookie:
raise ValueError("Quark 适配器需要 cookie 配置")
# 运行期缓存
self._dir_fid_cache: Dict[str, str] = {f"{self.root_fid}:": self.root_fid}
self._children_cache: Dict[str, List[Dict[str, Any]]] = {}
# UA 与超时
self._ua = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) quark-cloud-drive/2.5.20 Chrome/100.0.4896.160 "
"Electron/18.3.5.4-b478491100 Safari/537.36 Channel/pckk_other_ch"
)
self._timeout = 30.0
# -----------------
# 工具与通用请求
# -----------------
def get_effective_root(self, sub_path: str | None) -> str:
return self.root_fid
async def _request(
self,
method: str,
pathname: str,
*,
json: Any | None = None,
params: Dict[str, str] | None = None,
) -> Any:
headers = {
"Cookie": self._safe_cookie(self.cookie),
"Accept": "application/json, text/plain, */*",
"Referer": REFERER,
"User-Agent": self._ua,
}
query = {"pr": PR, "fr": "pc"}
if params:
query.update(params)
url = f"{API_BASE}{pathname}"
async with httpx.AsyncClient(timeout=self._timeout) as client:
resp = await client.request(method, url, headers=headers, params=query, json=json)
# 更新运行期 cookie(若返回 __puus/__pus
try:
for key in ("__puus", "__pus"):
v = resp.cookies.get(key)
if v:
# 简单替换/追加到 self.cookie
self._set_cookie_kv(key, v)
except Exception:
pass
# 解析业务状态
data = None
try:
data = resp.json()
except Exception:
resp.raise_for_status()
return resp
status = data.get("status")
code = data.get("code")
msg = data.get("message") or ""
if (status is not None and status >= 400) or (code is not None and code != 0):
raise HTTPException(502, detail=f"Quark error status={status} code={code} msg={msg}")
return data
def _set_cookie_kv(self, key: str, value: str):
# 将指定键值写入 self.cookie(粗略字符串处理)
parts = [p.strip() for p in (self.cookie or "").replace("\r", "").replace("\n", "").split(";") if p.strip()]
found = False
for i, p in enumerate(parts):
if p.startswith(key + "="):
parts[i] = f"{key}={value}"
found = True
break
if not found:
parts.append(f"{key}={value}")
self.cookie = "; ".join(parts)
def _sanitize_cookie(self, cookie: str) -> str:
if not cookie:
return ""
# 去除换行与前后空白
cookie = cookie.replace("\r", "").replace("\n", "").strip()
# 统一分号分隔并去除多余空格/空段
parts = [p.strip() for p in cookie.split(";") if p.strip()]
return "; ".join(parts)
def _safe_cookie(self, cookie: str) -> str:
s = self._sanitize_cookie(cookie)
# 仅保留可见 ASCII (0x20-0x7E)
s = "".join(ch for ch in s if 32 <= ord(ch) <= 126)
return s
# -----------------
# 列表与路径解析
# -----------------
def _map_file_item(self, it: Dict[str, Any]) -> Dict[str, Any]:
# Quark/UC 列表项:file=true 表示文件;false 表示目录
is_dir = not bool(it.get("file", False))
updated_at_ms = int(it.get("updated_at", 0) or 0)
name = it.get("file_name") or it.get("filename") or it.get("name")
return {
"fid": it.get("fid"),
"name": name,
"is_dir": is_dir,
"size": 0 if is_dir else int(it.get("size", 0) or 0),
"mtime": updated_at_ms // 1000 if updated_at_ms else 0,
"type": "dir" if is_dir else "file",
}
async def _list_children(self, parent_fid: str) -> List[Dict[str, Any]]:
if parent_fid in self._children_cache:
return self._children_cache[parent_fid]
files: List[Dict[str, Any]] = []
page = 1
size = 100
total = None
while True:
qp = {"pdir_fid": parent_fid, "_size": str(size), "_page": str(page), "_fetch_total": "1"}
data = await self._request("GET", "/file/sort", params=qp)
d = (data or {}).get("data", {})
meta = (data or {}).get("metadata", {})
page_files = d.get("list", [])
files.extend(page_files)
if total is None:
total = meta.get("_total") or meta.get("total") or 0
if page * size >= int(total):
break
page += 1
mapped = [self._map_file_item(x) for x in files if (not self.only_list_video_file) or (not x.get("file")) or (x.get("category") == 1)]
self._children_cache[parent_fid] = mapped
return mapped
def _dir_cache_key(self, base_fid: str, rel: str) -> str:
return f"{base_fid}:{rel.strip('/')}"
async def _resolve_dir_fid_from(self, base_fid: str, rel: str) -> str:
key = rel.strip("/")
cache_key = self._dir_cache_key(base_fid, key)
if cache_key in self._dir_fid_cache:
return self._dir_fid_cache[cache_key]
if key == "":
self._dir_fid_cache[cache_key] = base_fid
return base_fid
parent_fid = base_fid
path_so_far = []
for seg in key.split("/"):
if seg == "":
continue
path_so_far.append(seg)
cache_key = self._dir_cache_key(base_fid, "/".join(path_so_far))
cached = self._dir_fid_cache.get(cache_key)
if cached:
parent_fid = cached
continue
children = await self._list_children(parent_fid)
found = next((c for c in children if c["is_dir"] and c["name"] == seg), None)
if not found:
raise FileNotFoundError(f"Directory not found: {seg}")
parent_fid = found["fid"]
self._dir_fid_cache[cache_key] = parent_fid
return parent_fid
async def _find_child(self, parent_fid: str, name: str) -> Optional[Dict[str, Any]]:
children = await self._list_children(parent_fid)
for it in children:
if it["name"] == name:
return it
return None
def _invalidate_children_cache(self, parent_fid: str):
if parent_fid in self._children_cache:
try:
del self._children_cache[parent_fid]
except Exception:
pass
# -----------------
# 目录与文件列表
# -----------------
async def list_dir(
self,
root: str,
rel: str,
page_num: int = 1,
page_size: int = 50,
sort_by: str = "name",
sort_order: str = "asc",
) -> Tuple[List[Dict], int]:
base_fid = root or self.root_fid
fid = await self._resolve_dir_fid_from(base_fid, rel)
items = await self._list_children(fid)
# 排序,目录优先
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
key = (not item["is_dir"],)
sf = sort_by.lower()
if sf == "name":
key += (item["name"].lower(),)
elif sf == "size":
key += (item["size"],)
elif sf == "mtime":
key += (item["mtime"],)
else:
key += (item["name"].lower(),)
return key
items.sort(key=get_sort_key, reverse=reverse)
total = len(items)
start = (page_num - 1) * page_size
end = start + page_size
return items[start:end], total
# -----------------
# 下载与流式下载
# -----------------
async def _get_download_url(self, fid: str) -> str:
data = await self._request("POST", "/file/download", json={"fids": [fid]})
arr = (data or {}).get("data", [])
if not arr:
raise HTTPException(502, detail="No download data returned by Quark")
url = arr[0].get("download_url") or arr[0].get("DownloadUrl")
if not url:
raise HTTPException(502, detail="No download_url returned by Quark")
return url
async def _get_transcoding_url(self, fid: str) -> Optional[str]:
try:
payload = {"fid": fid, "resolutions": "low,normal,high,super,2k,4k", "supports": "fmp4_av,m3u8,dolby_vision"}
data = await self._request("POST", "/file/v2/play/project", json=payload)
lst = (data or {}).get("data", {}).get("video_list", [])
for item in lst:
vi = item.get("video_info") or {}
url = vi.get("url")
if url:
return url
except Exception:
return None
return None
async def get_video_transcoding_url(self, fid: str) -> Optional[str]:
if not self.use_transcoding_address:
return None
return await self._get_transcoding_url(fid)
def _is_video_name(self, name: str) -> bool:
mime, _ = mimetypes.guess_type(name)
return bool(mime and mime.startswith("video/"))
def _download_headers(self) -> Dict[str, str]:
return {"Cookie": self._safe_cookie(self.cookie), "User-Agent": self._ua, "Referer": REFERER}
async def read_file(self, root: str, rel: str) -> bytes:
if not rel or rel.endswith("/"):
raise IsADirectoryError("Path is a directory")
parent = rel.rsplit("/", 1)[0] if "/" in rel else ""
name = rel.rsplit("/", 1)[-1]
base_fid = root or self.root_fid
parent_fid = await self._resolve_dir_fid_from(base_fid, parent)
it = await self._find_child(parent_fid, name)
if not it or it["is_dir"]:
raise FileNotFoundError(rel)
url = await self._get_download_url(it["fid"])
headers = self._download_headers()
async with httpx.AsyncClient(timeout=None, follow_redirects=True) as client:
resp = await client.get(url, headers=headers)
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
return resp.content
async def read_file_range(self, root: str, rel: str, start: int, end: Optional[int] = None) -> bytes:
if not rel or rel.endswith("/"):
raise IsADirectoryError("Path is a directory")
parent = rel.rsplit("/", 1)[0] if "/" in rel else ""
name = rel.rsplit("/", 1)[-1]
base_fid = root or self.root_fid
parent_fid = await self._resolve_dir_fid_from(base_fid, parent)
it = await self._find_child(parent_fid, name)
if not it or it["is_dir"]:
raise FileNotFoundError(rel)
url = await self._get_download_url(it["fid"])
headers = dict(self._download_headers())
headers["Range"] = f"bytes={start}-" if end is None else f"bytes={start}-{end}"
async with httpx.AsyncClient(timeout=self._timeout, follow_redirects=True) as client:
resp = await client.get(url, headers=headers)
if resp.status_code == 404:
raise FileNotFoundError(rel)
if resp.status_code == 416:
raise HTTPException(416, detail="Requested Range Not Satisfiable")
resp.raise_for_status()
return resp.content
async def stream_file(self, root: str, rel: str, range_header: str | None):
if not rel or rel.endswith("/"):
raise IsADirectoryError("Path is a directory")
parent = rel.rsplit("/", 1)[0] if "/" in rel else ""
name = rel.rsplit("/", 1)[-1]
base_fid = root or self.root_fid
parent_fid = await self._resolve_dir_fid_from(base_fid, parent)
it = await self._find_child(parent_fid, name)
if not it or it["is_dir"]:
raise FileNotFoundError(rel)
url = await self._get_download_url(it["fid"])
if self.use_transcoding_address and self._is_video_name(name):
tr = await self._get_transcoding_url(it["fid"])
if tr:
url = tr
dl_headers = self._download_headers()
# 预获取大小/是否支持范围
total_size: Optional[int] = None
async with httpx.AsyncClient(timeout=self._timeout, follow_redirects=True) as client:
try:
head_resp = await client.head(url, headers=dl_headers)
if head_resp.status_code == 200:
cl = head_resp.headers.get("Content-Length")
if cl and cl.isdigit():
total_size = int(cl)
except Exception:
pass
mime, _ = mimetypes.guess_type(rel)
content_type = mime or "application/octet-stream"
# 解析 Range
start = 0
end: Optional[int] = None
status_code = 200
if range_header and range_header.startswith("bytes="):
status_code = 206
part = range_header.split("=", 1)[1]
s, e = part.split("-", 1)
if s.strip():
start = int(s)
if e.strip():
end = int(e)
if total_size is not None and end is None and status_code == 206:
end = total_size - 1
if end is not None and total_size is not None and end >= total_size:
end = total_size - 1
if total_size is not None and start >= total_size:
raise HTTPException(416, detail="Requested Range Not Satisfiable")
resp_headers: Dict[str, str] = {"Accept-Ranges": "bytes", "Content-Type": content_type}
if status_code == 206 and total_size is not None and end is not None:
resp_headers["Content-Range"] = f"bytes {start}-{end}/{total_size}"
resp_headers["Content-Length"] = str(end - start + 1)
elif total_size is not None:
resp_headers["Content-Length"] = str(total_size)
async def iterator():
headers = dict(dl_headers)
if status_code == 206 and end is not None:
headers["Range"] = f"bytes={start}-{end}"
async with httpx.AsyncClient(timeout=None, follow_redirects=True) as client:
async with client.stream("GET", url, headers=headers) as resp:
if resp.status_code in (404, 416):
await resp.aclose()
raise HTTPException(resp.status_code, detail="Upstream not available")
async for chunk in resp.aiter_bytes():
if chunk:
yield chunk
return StreamingResponse(iterator(), status_code=status_code, headers=resp_headers, media_type=content_type)
# -----------------
# 上传(大文件分片)
# -----------------
@staticmethod
def _md5_hex(b: bytes) -> str:
return hashlib.md5(b).hexdigest()
@staticmethod
def _sha1_hex(b: bytes) -> str:
return hashlib.sha1(b).hexdigest()
def _guess_mime(self, name: str) -> str:
mime, _ = mimetypes.guess_type(name)
return mime or "application/octet-stream"
async def _upload_pre(self, filename: str, size: int, parent_fid: str) -> Dict[str, Any]:
now_ms = int(time.time() * 1000)
body = {
"ccp_hash_update": True,
"dir_name": "",
"file_name": filename,
"format_type": self._guess_mime(filename),
"l_created_at": now_ms,
"l_updated_at": now_ms,
"pdir_fid": parent_fid,
"size": size,
}
data = await self._request("POST", "/file/upload/pre", json=body)
return data
async def write_file(self, root: str, rel: str, data: bytes):
async def gen():
yield data
return await self.write_file_stream(root, rel, gen())
async def write_upload_file(self, root: str, rel: str, file_obj, filename: str | None, file_size: int | None = None, content_type: str | None = None):
if not rel or rel.endswith("/"):
raise HTTPException(400, detail="Invalid file path")
parent = rel.rsplit("/", 1)[0] if "/" in rel else ""
name = filename or rel.rsplit("/", 1)[-1]
base_fid = root or self.root_fid
parent_fid = await self._resolve_dir_fid_from(base_fid, parent)
md5 = hashlib.md5()
sha1 = hashlib.sha1()
total = 0
try:
if callable(getattr(file_obj, "seek", None)):
file_obj.seek(0)
except Exception:
pass
while True:
chunk = file_obj.read(1024 * 1024)
if not chunk:
break
total += len(chunk)
md5.update(chunk)
sha1.update(chunk)
md5_hex = md5.hexdigest()
sha1_hex = sha1.hexdigest()
# 预上传,拿到上传信息
pre_resp = await self._upload_pre(name, total, parent_fid)
pre_data = pre_resp.get("data", {})
# hash 秒传
hash_body = {"md5": md5_hex, "sha1": sha1_hex, "task_id": pre_data.get("task_id")}
hash_resp = await self._request("POST", "/file/update/hash", json=hash_body)
if (hash_resp.get("data") or {}).get("finish") is True:
self._invalidate_children_cache(parent_fid)
return {"size": total}
# 分片上传
part_size = int((pre_resp.get("metadata") or {}).get("part_size") or 0)
if part_size <= 0:
raise HTTPException(502, detail="Invalid part_size from Quark")
bucket = pre_data.get("bucket")
obj_key = pre_data.get("obj_key")
upload_id = pre_data.get("upload_id")
upload_url = pre_data.get("upload_url")
if not (bucket and obj_key and upload_id and upload_url):
raise HTTPException(502, detail="Upload pre missing fields")
try:
upload_host = upload_url.split("://", 1)[1]
except Exception:
upload_host = upload_url
base_url = f"https://{bucket}.{upload_host}/{obj_key}"
try:
if callable(getattr(file_obj, "seek", None)):
file_obj.seek(0)
except Exception:
pass
etags: List[str] = []
oss_ua = "aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit"
async with httpx.AsyncClient(timeout=None, follow_redirects=True) as client:
part_number = 1
left = total
while left > 0:
sz = min(part_size, left)
data_bytes = file_obj.read(sz)
if len(data_bytes) != sz:
raise IOError("Failed to read part bytes")
now_str = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
auth_meta = (
"PUT\n\n"
f"{self._guess_mime(name)}\n"
f"{now_str}\n"
f"x-oss-date:{now_str}\n"
f"x-oss-user-agent:{oss_ua}\n"
f"/{bucket}/{obj_key}?partNumber={part_number}&uploadId={upload_id}"
)
auth_req_body = {"auth_info": pre_data.get("auth_info"), "auth_meta": auth_meta, "task_id": pre_data.get("task_id")}
auth_resp = await self._request("POST", "/file/upload/auth", json=auth_req_body)
auth_key = (auth_resp.get("data") or {}).get("auth_key")
if not auth_key:
raise HTTPException(502, detail="upload/auth missing auth_key")
put_headers = {
"Authorization": auth_key,
"Content-Type": self._guess_mime(name),
"Referer": REFERER + "/",
"x-oss-date": now_str,
"x-oss-user-agent": oss_ua,
}
put_url = f"{base_url}?partNumber={part_number}&uploadId={upload_id}"
put_resp = await client.put(put_url, headers=put_headers, content=data_bytes)
if put_resp.status_code != 200:
raise HTTPException(502, detail=f"Upload part failed status={put_resp.status_code} text={put_resp.text}")
etag = put_resp.headers.get("Etag", "")
etags.append(etag)
left -= sz
part_number += 1
parts_xml = [f"<Part>\n<PartNumber>{i+1}</PartNumber>\n<ETag>{etags[i]}</ETag>\n</Part>\n" for i in range(len(etags))]
body_xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<CompleteMultipartUpload>\n" + "".join(parts_xml) + "</CompleteMultipartUpload>"
content_md5 = base64.b64encode(hashlib.md5(body_xml.encode("utf-8")).digest()).decode("ascii")
callback = pre_data.get("callback") or {}
try:
import json as _json
callback_b64 = base64.b64encode(_json.dumps(callback).encode("utf-8")).decode("ascii")
except Exception:
callback_b64 = ""
now_str = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
auth_meta_commit = (
"POST\n"
f"{content_md5}\n"
"application/xml\n"
f"{now_str}\n"
f"x-oss-callback:{callback_b64}\n"
f"x-oss-date:{now_str}\n"
f"x-oss-user-agent:{oss_ua}\n"
f"/{bucket}/{obj_key}?uploadId={upload_id}"
)
auth_commit_resp = await self._request("POST", "/file/upload/auth", json={"auth_info": pre_data.get("auth_info"), "auth_meta": auth_meta_commit, "task_id": pre_data.get("task_id")})
auth_key_commit = (auth_commit_resp.get("data") or {}).get("auth_key")
if not auth_key_commit:
raise HTTPException(502, detail="upload/auth(commit) missing auth_key")
async with httpx.AsyncClient(timeout=None, follow_redirects=True) as client:
commit_headers = {
"Authorization": auth_key_commit,
"Content-MD5": content_md5,
"Content-Type": "application/xml",
"Referer": REFERER + "/",
"x-oss-callback": callback_b64,
"x-oss-date": now_str,
"x-oss-user-agent": oss_ua,
}
commit_url = f"{base_url}?uploadId={upload_id}"
r = await client.post(commit_url, headers=commit_headers, content=body_xml.encode("utf-8"))
if r.status_code != 200:
raise HTTPException(502, detail=f"Upload commit failed status={r.status_code} text={r.text}")
await self._request("POST", "/file/upload/finish", json={"obj_key": obj_key, "task_id": pre_data.get("task_id")})
try:
await asyncio.sleep(1.0)
except Exception:
pass
self._invalidate_children_cache(parent_fid)
return {"size": total}
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
if not rel or rel.endswith("/"):
raise HTTPException(400, detail="Invalid file path")
parent = rel.rsplit("/", 1)[0] if "/" in rel else ""
name = rel.rsplit("/", 1)[-1]
base_fid = root or self.root_fid
parent_fid = await self._resolve_dir_fid_from(base_fid, parent)
# 将数据落盘到临时文件,同时计算 MD5/SHA1
import tempfile
md5 = hashlib.md5()
sha1 = hashlib.sha1()
total = 0
with tempfile.NamedTemporaryFile(delete=False) as tf:
tmp_path = tf.name
try:
async for chunk in data_iter:
if not chunk:
continue
total += len(chunk)
md5.update(chunk)
sha1.update(chunk)
tf.write(chunk)
finally:
tf.flush()
md5_hex = md5.hexdigest()
sha1_hex = sha1.hexdigest()
# 预上传,拿到上传信息
pre_resp = await self._upload_pre(name, total, parent_fid)
pre_data = pre_resp.get("data", {})
# hash 秒传
hash_body = {"md5": md5_hex, "sha1": sha1_hex, "task_id": pre_data.get("task_id")}
hash_resp = await self._request("POST", "/file/update/hash", json=hash_body)
if (hash_resp.get("data") or {}).get("finish") is True:
try:
os.unlink(tmp_path)
except Exception:
pass
# 刷新父目录缓存
self._invalidate_children_cache(parent_fid)
return total
# 分片上传
part_size = int((pre_resp.get("metadata") or {}).get("part_size") or 0)
if part_size <= 0:
raise HTTPException(502, detail="Invalid part_size from Quark")
bucket = pre_data.get("bucket")
obj_key = pre_data.get("obj_key")
upload_id = pre_data.get("upload_id")
upload_url = pre_data.get("upload_url")
if not (bucket and obj_key and upload_id and upload_url):
raise HTTPException(502, detail="Upload pre missing fields")
# 计算 host 与基础 URL
try:
upload_host = upload_url.split("://", 1)[1]
except Exception:
upload_host = upload_url
base_url = f"https://{bucket}.{upload_host}/{obj_key}"
# 分片循环
etags: List[str] = []
oss_ua = "aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit"
async with httpx.AsyncClient(timeout=None, follow_redirects=True) as client:
with open(tmp_path, "rb") as rf:
part_number = 1
left = total
while left > 0:
sz = min(part_size, left)
data_bytes = rf.read(sz)
if len(data_bytes) != sz:
raise IOError("Failed to read part bytes")
now_str = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
# 申请签名
auth_meta = (
"PUT\n\n"
f"{self._guess_mime(name)}\n"
f"{now_str}\n"
f"x-oss-date:{now_str}\n"
f"x-oss-user-agent:{oss_ua}\n"
f"/{bucket}/{obj_key}?partNumber={part_number}&uploadId={upload_id}"
)
auth_req_body = {"auth_info": pre_data.get("auth_info"), "auth_meta": auth_meta, "task_id": pre_data.get("task_id")}
auth_resp = await self._request("POST", "/file/upload/auth", json=auth_req_body)
auth_key = (auth_resp.get("data") or {}).get("auth_key")
if not auth_key:
raise HTTPException(502, detail="upload/auth missing auth_key")
put_headers = {
"Authorization": auth_key,
"Content-Type": self._guess_mime(name),
"Referer": REFERER + "/",
"x-oss-date": now_str,
"x-oss-user-agent": oss_ua,
}
put_url = f"{base_url}?partNumber={part_number}&uploadId={upload_id}"
put_resp = await client.put(put_url, headers=put_headers, content=data_bytes)
if put_resp.status_code != 200:
raise HTTPException(502, detail=f"Upload part failed status={put_resp.status_code} text={put_resp.text}")
etag = put_resp.headers.get("Etag", "")
etags.append(etag)
left -= sz
part_number += 1
# 组合 commit xml
parts_xml = [f"<Part>\n<PartNumber>{i+1}</PartNumber>\n<ETag>{etags[i]}</ETag>\n</Part>\n" for i in range(len(etags))]
body_xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<CompleteMultipartUpload>\n" + "".join(parts_xml) + "</CompleteMultipartUpload>"
content_md5 = base64.b64encode(hashlib.md5(body_xml.encode("utf-8")).digest()).decode("ascii")
callback = pre_data.get("callback") or {}
try:
import json as _json
callback_b64 = base64.b64encode(_json.dumps(callback).encode("utf-8")).decode("ascii")
except Exception:
callback_b64 = ""
now_str = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
auth_meta_commit = (
"POST\n"
f"{content_md5}\n"
"application/xml\n"
f"{now_str}\n"
f"x-oss-callback:{callback_b64}\n"
f"x-oss-date:{now_str}\n"
f"x-oss-user-agent:{oss_ua}\n"
f"/{bucket}/{obj_key}?uploadId={upload_id}"
)
auth_commit_resp = await self._request("POST", "/file/upload/auth", json={"auth_info": pre_data.get("auth_info"), "auth_meta": auth_meta_commit, "task_id": pre_data.get("task_id")})
auth_key_commit = (auth_commit_resp.get("data") or {}).get("auth_key")
if not auth_key_commit:
raise HTTPException(502, detail="upload/auth(commit) missing auth_key")
async with httpx.AsyncClient(timeout=None, follow_redirects=True) as client:
commit_headers = {
"Authorization": auth_key_commit,
"Content-MD5": content_md5,
"Content-Type": "application/xml",
"Referer": REFERER + "/",
"x-oss-callback": callback_b64,
"x-oss-date": now_str,
"x-oss-user-agent": oss_ua,
}
commit_url = f"{base_url}?uploadId={upload_id}"
r = await client.post(commit_url, headers=commit_headers, content=body_xml.encode("utf-8"))
if r.status_code != 200:
raise HTTPException(502, detail=f"Upload commit failed status={r.status_code} text={r.text}")
# finish
await self._request("POST", "/file/upload/finish", json={"obj_key": obj_key, "task_id": pre_data.get("task_id")})
# 端合并存在轻微延迟,等待再刷新缓存
try:
await asyncio.sleep(1.0)
except Exception:
pass
try:
os.unlink(tmp_path)
except Exception:
pass
# 失效父目录缓存,确保后续列表可见
self._invalidate_children_cache(parent_fid)
return total
# -----------------
# 基本文件操作
# -----------------
async def mkdir(self, root: str, rel: str):
if not rel or rel == "/":
raise HTTPException(400, detail="Cannot create root")
parent = rel.rstrip("/")
parent_rel, name = (parent.rsplit("/", 1) if "/" in parent else ("", parent))
if not name:
raise HTTPException(400, detail="Invalid directory name")
pdir = await self._resolve_dir_fid_from(root or self.root_fid, parent_rel)
await self._request("POST", "/file", json={"dir_init_lock": False, "dir_path": "", "file_name": name, "pdir_fid": pdir})
self._invalidate_children_cache(pdir)
async def delete(self, root: str, rel: str):
# 解析对象 fid + 父目录,用于失效缓存
base_fid = root or self.root_fid
if rel == "" or rel.endswith("/"):
parent_rel = rel.rstrip("/")
target_fid = await self._resolve_dir_fid_from(base_fid, parent_rel)
parent_of_target = await self._resolve_dir_fid_from(base_fid, (parent_rel.rsplit("/", 1)[0] if "/" in parent_rel else ""))
else:
parent_rel, name = (rel.rsplit("/", 1) if "/" in rel else ("", rel))
parent_of_target = await self._resolve_dir_fid_from(base_fid, parent_rel)
it = await self._find_child(parent_of_target, name)
if not it:
return
target_fid = it["fid"]
await self._request("POST", "/file/delete", json={"action_type": 1, "exclude_fids": [], "filelist": [target_fid]})
self._invalidate_children_cache(parent_of_target)
async def move(self, root: str, src_rel: str, dst_rel: str):
# 支持跨目录与重命名:先移动到父目录,后重命名(若需要)
src_parent_rel, src_name = (src_rel.rsplit("/", 1) if "/" in src_rel else ("", src_rel))
dst_parent_rel, dst_name = (dst_rel.rsplit("/", 1) if "/" in dst_rel else ("", dst_rel))
base_fid = root or self.root_fid
src_parent_fid = await self._resolve_dir_fid_from(base_fid, src_parent_rel)
obj = await self._find_child(src_parent_fid, src_name)
if not obj:
raise FileNotFoundError(src_rel)
dst_parent_fid = await self._resolve_dir_fid_from(base_fid, dst_parent_rel)
if src_parent_fid != dst_parent_fid:
await self._request("POST", "/file/move", json={"action_type": 1, "exclude_fids": [], "filelist": [obj["fid"]], "to_pdir_fid": dst_parent_fid})
self._invalidate_children_cache(src_parent_fid)
self._invalidate_children_cache(dst_parent_fid)
if obj["name"] != dst_name:
await self._request("POST", "/file/rename", json={"fid": obj["fid"], "file_name": dst_name})
self._invalidate_children_cache(dst_parent_fid)
async def rename(self, root: str, src_rel: str, dst_rel: str):
src_parent_rel, src_name = (src_rel.rsplit("/", 1) if "/" in src_rel else ("", src_rel))
base_fid = root or self.root_fid
src_parent_fid = await self._resolve_dir_fid_from(base_fid, src_parent_rel)
obj = await self._find_child(src_parent_fid, src_name)
if not obj:
raise FileNotFoundError(src_rel)
dst_name = dst_rel.rsplit("/", 1)[-1]
await self._request("POST", "/file/rename", json={"fid": obj["fid"], "file_name": dst_name})
self._invalidate_children_cache(src_parent_fid)
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
raise NotImplementedError("QuarkOpen does not support copy via open API")
# -----------------
# STAT / EXISTS / 辅助
# -----------------
async def stat_file(self, root: str, rel: str):
# 通过父目录列表获取元数据
base_fid = root or self.root_fid
if rel == "" or rel.endswith("/"):
# 目录
fid = await self._resolve_dir_fid_from(base_fid, rel.rstrip("/"))
return {"name": rel.rstrip("/").split("/")[-1] if rel else "", "is_dir": True, "size": 0, "mtime": 0, "type": "dir", "fid": fid}
parent_rel, name = (rel.rsplit("/", 1) if "/" in rel else ("", rel))
parent_fid = await self._resolve_dir_fid_from(base_fid, parent_rel)
it = await self._find_child(parent_fid, name)
if not it:
raise FileNotFoundError(rel)
return it
async def exists(self, root: str, rel: str) -> bool:
try:
base_fid = root or self.root_fid
if rel == "" or rel.endswith("/"):
await self._resolve_dir_fid_from(base_fid, rel.rstrip("/"))
return True
parent_rel, name = (rel.rsplit("/", 1) if "/" in rel else ("", rel))
parent_fid = await self._resolve_dir_fid_from(base_fid, parent_rel)
it = await self._find_child(parent_fid, name)
return it is not None
except FileNotFoundError:
return False
async def stat_path(self, root: str, rel: str):
# 用于 move/copy 前的预检查调试
try:
base_fid = root or self.root_fid
if rel == "" or rel.endswith("/"):
fid = await self._resolve_dir_fid_from(base_fid, rel.rstrip("/"))
return {"exists": True, "is_dir": True, "path": rel, "fid": fid}
parent_rel, name = (rel.rsplit("/", 1) if "/" in rel else ("", rel))
parent_fid = await self._resolve_dir_fid_from(base_fid, parent_rel)
it = await self._find_child(parent_fid, name)
if it:
return {"exists": True, "is_dir": it["is_dir"], "path": rel, "fid": it["fid"]}
return {"exists": False, "is_dir": None, "path": rel}
except FileNotFoundError:
return {"exists": False, "is_dir": None, "path": rel}
async def _resolve_target_fid(self, rel: str, *, base_fid: Optional[str] = None) -> str:
base = base_fid or self.root_fid
if rel == "" or rel.endswith("/"):
return await self._resolve_dir_fid_from(base, rel.rstrip("/"))
parent_rel, name = (rel.rsplit("/", 1) if "/" in rel else ("", rel))
parent_fid = await self._resolve_dir_fid_from(base, parent_rel)
it = await self._find_child(parent_fid, name)
if not it:
raise FileNotFoundError(rel)
return it["fid"]
ADAPTER_TYPE = "quark"
CONFIG_SCHEMA = [
{"key": "cookie", "label": "Cookie", "type": "password", "required": True, "placeholder": "从 pan.quark.cn 复制"},
{"key": "root_fid", "label": "根 FID", "type": "string", "required": False, "default": "0"},
{"key": "use_transcoding_address", "label": "视频转码直链", "type": "boolean", "required": False, "default": False},
{"key": "only_list_video_file", "label": "仅列出视频文件", "type": "boolean", "required": False, "default": False},
]
def ADAPTER_FACTORY(rec: StorageAdapter) -> BaseAdapter:
return QuarkAdapter(rec)
+347
View File
@@ -0,0 +1,347 @@
import asyncio
import mimetypes
from datetime import datetime
from typing import List, Dict, Tuple, AsyncIterator
from urllib.parse import quote
import aioboto3
from botocore.exceptions import ClientError
from fastapi import HTTPException
from fastapi.responses import StreamingResponse
from models import StorageAdapter
class S3Adapter:
"""S3 兼容对象存储适配器"""
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config
self.bucket_name = cfg.get("bucket_name")
self.aws_access_key_id = cfg.get("access_key_id")
self.aws_secret_access_key = cfg.get("secret_access_key")
self.region_name = cfg.get("region_name")
self.endpoint_url = cfg.get("endpoint_url")
self.root = cfg.get("root", "").strip("/")
if not all([self.bucket_name, self.aws_access_key_id, self.aws_secret_access_key]):
raise ValueError(
"S3 适配器需要 bucket_name, access_key_id, 和 secret_access_key")
self.session = aioboto3.Session(
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
region_name=self.region_name,
)
def get_effective_root(self, sub_path: str | None) -> str:
"""获取 S3 中的有效根路径 (key prefix)"""
if sub_path:
return f"{self.root}/{sub_path.strip('/')}".strip("/")
return self.root
def _get_s3_key(self, rel_path: str) -> str:
"""将相对路径转换为 S3 key"""
rel_path = rel_path.strip("/")
if self.root:
return f"{self.root}/{rel_path}"
return rel_path
def _get_client(self):
return self.session.client("s3", endpoint_url=self.endpoint_url)
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
prefix = self._get_s3_key(rel)
if prefix and not prefix.endswith("/"):
prefix += "/"
all_items = []
async with self._get_client() as s3:
paginator = s3.get_paginator("list_objects_v2")
async for result in paginator.paginate(Bucket=self.bucket_name, Prefix=prefix, Delimiter="/"):
# 添加子目录
for common_prefix in result.get("CommonPrefixes", []):
dir_name = common_prefix.get(
"Prefix").removeprefix(prefix).strip("/")
if dir_name:
all_items.append({
"name": dir_name,
"is_dir": True,
"size": 0,
"mtime": 0,
"type": "dir",
})
# 添加文件
for content in result.get("Contents", []):
file_key = content.get("Key")
if file_key == prefix: # 忽略目录本身
continue
file_name = file_key.removeprefix(prefix)
if file_name:
all_items.append({
"name": file_name,
"is_dir": False,
"size": content.get("Size", 0),
"mtime": int(content.get("LastModified", datetime.now()).timestamp()),
"type": "file",
})
# 在内存中排序和分页
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
key = (not item["is_dir"],)
sort_field = sort_by.lower()
if sort_field == "name":
key += (item["name"].lower(),)
elif sort_field == "size":
key += (item["size"],)
elif sort_field == "mtime":
key += (item["mtime"],)
else:
key += (item["name"].lower(),)
return key
all_items.sort(key=get_sort_key, reverse=reverse)
total_count = len(all_items)
start_idx = (page_num - 1) * page_size
end_idx = start_idx + page_size
return all_items[start_idx:end_idx], total_count
async def read_file(self, root: str, rel: str) -> bytes:
key = self._get_s3_key(rel)
async with self._get_client() as s3:
try:
resp = await s3.get_object(Bucket=self.bucket_name, Key=key)
return await resp["Body"].read()
except ClientError as e:
if e.response["Error"]["Code"] == "NoSuchKey":
raise FileNotFoundError(rel)
raise
async def write_file(self, root: str, rel: str, data: bytes):
key = self._get_s3_key(rel)
async with self._get_client() as s3:
await s3.put_object(Bucket=self.bucket_name, Key=key, Body=data)
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
key = self._get_s3_key(rel)
MIN_PART_SIZE = 5 * 1024 * 1024
async with self._get_client() as s3:
mpu = await s3.create_multipart_upload(Bucket=self.bucket_name, Key=key)
upload_id = mpu['UploadId']
parts = []
part_number = 1
total_size = 0
buffer = bytearray()
try:
async for chunk in data_iter:
if not chunk:
continue
buffer.extend(chunk)
while len(buffer) >= MIN_PART_SIZE:
part_data = buffer[:MIN_PART_SIZE]
del buffer[:MIN_PART_SIZE]
part = await s3.upload_part(
Bucket=self.bucket_name,
Key=key,
PartNumber=part_number,
UploadId=upload_id,
Body=part_data
)
parts.append({'PartNumber': part_number, 'ETag': part['ETag']})
total_size += len(part_data)
part_number += 1
if buffer:
part = await s3.upload_part(
Bucket=self.bucket_name,
Key=key,
PartNumber=part_number,
UploadId=upload_id,
Body=bytes(buffer)
)
parts.append({'PartNumber': part_number, 'ETag': part['ETag']})
total_size += len(buffer)
await s3.complete_multipart_upload(
Bucket=self.bucket_name,
Key=key,
UploadId=upload_id,
MultipartUpload={'Parts': parts}
)
except Exception as e:
await s3.abort_multipart_upload(
Bucket=self.bucket_name,
Key=key,
UploadId=upload_id
)
raise IOError(f"S3 stream upload failed: {e}") from e
return total_size
async def mkdir(self, root: str, rel: str):
key = self._get_s3_key(rel)
if not key.endswith("/"):
key += "/"
async with self._get_client() as s3:
await s3.put_object(Bucket=self.bucket_name, Key=key, Body=b"")
async def delete(self, root: str, rel: str):
key = self._get_s3_key(rel)
async with self._get_client() as s3:
is_dir_like = False
try:
head = await s3.head_object(Bucket=self.bucket_name, Key=key)
if head['ContentLength'] == 0 and key.endswith('/'):
is_dir_like = True
except ClientError as e:
if e.response['Error']['Code'] != '404':
raise
# 如果是目录,删除目录下的所有对象
if is_dir_like or not await self.stat_file(root, rel):
dir_key = key if key.endswith('/') else key + '/'
paginator = s3.get_paginator("list_objects_v2")
objects_to_delete = []
async for result in paginator.paginate(Bucket=self.bucket_name, Prefix=dir_key):
for content in result.get("Contents", []):
objects_to_delete.append({"Key": content["Key"]})
if objects_to_delete:
await s3.delete_objects(Bucket=self.bucket_name, Delete={"Objects": objects_to_delete})
# 如果是文件,直接删除
else:
await s3.delete_object(Bucket=self.bucket_name, Key=key)
async def move(self, root: str, src_rel: str, dst_rel: str):
await self.copy(root, src_rel, dst_rel, overwrite=True)
await self.delete(root, src_rel)
async def rename(self, root: str, src_rel: str, dst_rel: str):
await self.move(root, src_rel, dst_rel)
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
src_key = self._get_s3_key(src_rel)
dst_key = self._get_s3_key(dst_rel)
async with self._get_client() as s3:
if not overwrite:
try:
await s3.head_object(Bucket=self.bucket_name, Key=dst_key)
raise FileExistsError(dst_rel)
except ClientError as e:
if e.response["Error"]["Code"] != "404":
raise
copy_source = {"Bucket": self.bucket_name, "Key": src_key}
await s3.copy_object(CopySource=copy_source, Bucket=self.bucket_name, Key=dst_key)
async def stat_file(self, root: str, rel: str):
key = self._get_s3_key(rel)
async with self._get_client() as s3:
try:
head = await s3.head_object(Bucket=self.bucket_name, Key=key)
return {
"name": rel.split("/")[-1],
"is_dir": False,
"size": head["ContentLength"],
"mtime": int(head["LastModified"].timestamp()),
"type": "file",
}
except ClientError as e:
if e.response["Error"]["Code"] == "404":
# 检查是否为一个 "目录"
dir_key = key if key.endswith('/') else key + '/'
resp = await s3.list_objects_v2(Bucket=self.bucket_name, Prefix=dir_key, MaxKeys=1)
if resp.get('KeyCount', 0) > 0:
return {
"name": rel.split("/")[-1],
"is_dir": True,
"size": 0,
"mtime": 0,
"type": "dir",
}
raise FileNotFoundError(rel)
raise
async def stream_file(self, root: str, rel: str, range_header: str | None):
key = self._get_s3_key(rel)
async with self._get_client() as s3:
try:
head = await s3.head_object(Bucket=self.bucket_name, Key=key)
file_size = head["ContentLength"]
content_type = head.get("ContentType", mimetypes.guess_type(key)[
0] or "application/octet-stream")
except ClientError as e:
if e.response["Error"]["Code"] == "404":
raise HTTPException(
status_code=404, detail="File not found")
raise
start = 0
end = file_size - 1
status = 200
headers = {
"Accept-Ranges": "bytes",
"Content-Type": content_type,
"Content-Length": str(file_size),
"Content-Disposition": f"inline; filename=\"{quote(rel.split('/')[-1])}\""
}
if range_header:
range_val = range_header.strip().partition("=")[2]
s, _, e = range_val.partition("-")
try:
start = int(s) if s else 0
end = int(e) if e else file_size - 1
if start >= file_size or end >= file_size or start > end:
raise HTTPException(
status_code=416, detail="Requested Range Not Satisfiable")
status = 206
headers["Content-Length"] = str(end - start + 1)
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
except ValueError:
raise HTTPException(
status_code=400, detail="Invalid Range header")
range_arg = f"bytes={start}-{end}"
async def iterator():
try:
resp = await s3.get_object(Bucket=self.bucket_name, Key=key, Range=range_arg)
body = resp["Body"]
while chunk := await body.read(65536):
yield chunk
except Exception as e:
raise
return StreamingResponse(iterator(), status_code=status, headers=headers, media_type=content_type)
ADAPTER_TYPE = "s3"
CONFIG_SCHEMA = [
{"key": "bucket_name", "label": "Bucket 名称",
"type": "string", "required": True},
{"key": "access_key_id", "label": "Access Key ID",
"type": "string", "required": True},
{"key": "secret_access_key", "label": "Secret Access Key",
"type": "password", "required": True},
{"key": "region_name", "label": "区域 (Region)", "type": "string",
"required": False, "placeholder": "例如 us-east-1"},
{"key": "endpoint_url", "label": "Endpoint URL", "type": "string",
"required": False, "placeholder": "对于 S3 兼容存储, 例如 https://minio.example.com"},
{"key": "root", "label": "根路径 (Root Path)", "type": "string",
"required": False, "placeholder": "在 bucket 内的路径前缀"},
]
def ADAPTER_FACTORY(rec): return S3Adapter(rec)
+473
View File
@@ -0,0 +1,473 @@
import asyncio
import mimetypes
import stat as statmod
from typing import List, Dict, Tuple, AsyncIterator, Optional
from fastapi import HTTPException
from fastapi.responses import StreamingResponse
import paramiko
from models import StorageAdapter
def _join_remote(root: str, rel: str) -> str:
root = (root or "/").rstrip("/") or "/"
rel = (rel or "").lstrip("/")
if not rel:
return root
return f"{root}/{rel}"
class SFTPAdapter:
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config
self.host: str = cfg.get("host")
self.port: int = int(cfg.get("port", 22))
self.username: str | None = cfg.get("username")
self.password: str | None = cfg.get("password")
self.timeout: int = int(cfg.get("timeout", 15))
self.root_path: str = cfg.get("root") # 必填
self.allow_unknown_host: bool = bool(cfg.get("allow_unknown_host", True))
if not self.host:
raise ValueError("SFTP adapter requires 'host'")
if not self.username or not self.password:
raise ValueError("SFTP adapter requires 'username' and 'password'")
if not self.root_path:
raise ValueError("SFTP adapter requires 'root'")
def get_effective_root(self, sub_path: str | None) -> str:
base = self.root_path.rstrip("/") or "/"
if sub_path:
return _join_remote(base, sub_path)
return base
def _connect(self) -> paramiko.SFTPClient:
ssh = paramiko.SSHClient()
if self.allow_unknown_host:
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(
hostname=self.host,
port=self.port,
username=self.username,
password=self.password,
timeout=self.timeout,
allow_agent=False,
look_for_keys=False,
)
return ssh.open_sftp()
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
path = _join_remote(root, rel)
def _do_list() -> List[Dict]:
sftp = self._connect()
try:
attrs = sftp.listdir_attr(path)
entries: List[Dict] = []
for a in attrs:
name = a.filename
is_dir = statmod.S_ISDIR(a.st_mode)
entries.append({
"name": name,
"is_dir": is_dir,
"size": 0 if is_dir else int(a.st_size or 0),
"mtime": int(a.st_mtime or 0),
"type": "dir" if is_dir else "file",
})
return entries
finally:
try:
sftp.close()
except Exception:
pass
entries = await asyncio.to_thread(_do_list)
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
key = (not item["is_dir"],)
f = sort_by.lower()
if f == "name":
key += (item["name"].lower(),)
elif f == "size":
key += (item.get("size", 0),)
elif f == "mtime":
key += (item.get("mtime", 0),)
else:
key += (item["name"].lower(),)
return key
entries.sort(key=get_sort_key, reverse=reverse)
total = len(entries)
start = (page_num - 1) * page_size
end = start + page_size
return entries[start:end], total
async def read_file(self, root: str, rel: str) -> bytes:
path = _join_remote(root, rel)
def _do_read() -> bytes:
sftp = self._connect()
try:
with sftp.open(path, "rb") as f:
return f.read()
except FileNotFoundError:
raise
except IOError as e:
if getattr(e, "errno", None) == 2:
raise FileNotFoundError(rel)
raise
finally:
try:
sftp.close()
except Exception:
pass
return await asyncio.to_thread(_do_read)
async def write_file(self, root: str, rel: str, data: bytes):
path = _join_remote(root, rel)
def _ensure_dirs(sftp: paramiko.SFTPClient, dir_path: str):
parts = [p for p in dir_path.strip("/").split("/") if p]
cur = "/"
for p in parts:
cur = _join_remote(cur, p)
try:
sftp.mkdir(cur)
except IOError:
# likely exists
pass
def _do_write():
sftp = self._connect()
try:
parent = "/" if "/" not in path.strip("/") else path.rsplit("/", 1)[0]
_ensure_dirs(sftp, parent)
with sftp.open(path, "wb") as f:
f.write(data)
finally:
try:
sftp.close()
except Exception:
pass
await asyncio.to_thread(_do_write)
async def write_upload_file(self, root: str, rel: str, file_obj, filename: str | None, file_size: int | None = None, content_type: str | None = None):
path = _join_remote(root, rel)
def _ensure_dirs(sftp: paramiko.SFTPClient, dir_path: str):
parts = [p for p in dir_path.strip("/").split("/") if p]
cur = "/"
for p in parts:
cur = _join_remote(cur, p)
try:
sftp.mkdir(cur)
except IOError:
pass
def _do_upload():
sftp = self._connect()
try:
parent = "/" if "/" not in path.strip("/") else path.rsplit("/", 1)[0]
_ensure_dirs(sftp, parent)
try:
if callable(getattr(file_obj, "seek", None)):
file_obj.seek(0)
except Exception:
pass
with sftp.open(path, "wb") as f:
import shutil
shutil.copyfileobj(file_obj, f)
finally:
try:
sftp.close()
except Exception:
pass
await asyncio.to_thread(_do_upload)
return {"size": file_size or 0}
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
buf = bytearray()
async for chunk in data_iter:
if chunk:
buf.extend(chunk)
await self.write_file(root, rel, bytes(buf))
return len(buf)
async def mkdir(self, root: str, rel: str):
path = _join_remote(root, rel)
def _do_mkdir():
sftp = self._connect()
try:
parts = [p for p in path.strip("/").split("/") if p]
cur = "/"
for p in parts:
cur = _join_remote(cur, p)
try:
sftp.mkdir(cur)
except IOError:
pass
finally:
try:
sftp.close()
except Exception:
pass
await asyncio.to_thread(_do_mkdir)
async def delete(self, root: str, rel: str):
path = _join_remote(root, rel)
def _do_delete():
sftp = self._connect()
try:
# Try file remove first
try:
sftp.remove(path)
return
except IOError:
pass
def _rm_tree(dp: str):
try:
for a in sftp.listdir_attr(dp):
child = _join_remote(dp, a.filename)
if statmod.S_ISDIR(a.st_mode):
_rm_tree(child)
else:
try:
sftp.remove(child)
except Exception:
pass
sftp.rmdir(dp)
except IOError:
pass
_rm_tree(path)
finally:
try:
sftp.close()
except Exception:
pass
await asyncio.to_thread(_do_delete)
async def move(self, root: str, src_rel: str, dst_rel: str):
src = _join_remote(root, src_rel)
dst = _join_remote(root, dst_rel)
def _do_move():
sftp = self._connect()
try:
# ensure dst parent exists
parent = "/" if "/" not in dst.strip("/") else dst.rsplit("/", 1)[0]
parts = [p for p in parent.strip("/").split("/") if p]
cur = "/"
for p in parts:
cur = _join_remote(cur, p)
try:
sftp.mkdir(cur)
except IOError:
pass
sftp.rename(src, dst)
finally:
try:
sftp.close()
except Exception:
pass
await asyncio.to_thread(_do_move)
async def rename(self, root: str, src_rel: str, dst_rel: str):
await self.move(root, src_rel, dst_rel)
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
src = _join_remote(root, src_rel)
dst = _join_remote(root, dst_rel)
def _is_dir() -> bool:
sftp = self._connect()
try:
st = sftp.stat(src)
return statmod.S_ISDIR(st.st_mode)
finally:
try:
sftp.close()
except Exception:
pass
if await asyncio.to_thread(_is_dir):
await self.mkdir(root, dst_rel)
children, _ = await self.list_dir(root, src_rel, page_num=1, page_size=10_000)
for ent in children:
child_src = f"{src_rel.rstrip('/')}/{ent['name']}"
child_dst = f"{dst_rel.rstrip('/')}/{ent['name']}"
await self.copy(root, child_src, child_dst, overwrite)
return
# file copy
data = await self.read_file(root, src_rel)
if not overwrite:
try:
await self.stat_file(root, dst_rel)
raise FileExistsError(dst_rel)
except FileNotFoundError:
pass
await self.write_file(root, dst_rel, data)
async def stat_file(self, root: str, rel: str):
path = _join_remote(root, rel)
def _do_stat():
sftp = self._connect()
try:
st = sftp.stat(path)
is_dir = statmod.S_ISDIR(st.st_mode)
info = {
"name": rel.split("/")[-1],
"is_dir": is_dir,
"size": 0 if is_dir else int(st.st_size or 0),
"mtime": int(st.st_mtime or 0),
"type": "dir" if is_dir else "file",
"path": path,
}
return info
except FileNotFoundError:
raise
except IOError as e:
if getattr(e, "errno", None) == 2:
raise FileNotFoundError(rel)
raise
finally:
try:
sftp.close()
except Exception:
pass
return await asyncio.to_thread(_do_stat)
async def exists(self, root: str, rel: str) -> bool:
try:
await self.stat_file(root, rel)
return True
except FileNotFoundError:
return False
except Exception:
return False
async def stream_file(self, root: str, rel: str, range_header: str | None):
path = _join_remote(root, rel)
def _get_stat():
sftp = self._connect()
try:
st = sftp.stat(path)
return int(st.st_size or 0)
finally:
try:
sftp.close()
except Exception:
pass
file_size = await asyncio.to_thread(_get_stat)
if file_size is None:
raise HTTPException(404, detail="File not found")
mime, _ = mimetypes.guess_type(rel)
content_type = mime or "application/octet-stream"
start = 0
end = file_size - 1
status = 200
headers = {
"Accept-Ranges": "bytes",
"Content-Type": content_type,
"Content-Length": str(file_size),
}
if range_header and range_header.startswith("bytes="):
try:
s, e = (range_header.removeprefix("bytes=").split("-", 1))
if s.strip():
start = int(s)
if e.strip():
end = int(e)
if start >= file_size:
raise HTTPException(416, detail="Requested Range Not Satisfiable")
if end >= file_size:
end = file_size - 1
status = 206
headers["Content-Length"] = str(end - start + 1)
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
except ValueError:
raise HTTPException(400, detail="Invalid Range header")
queue: asyncio.Queue[Optional[bytes]] = asyncio.Queue(maxsize=8)
def _worker():
sftp = self._connect()
try:
with sftp.open(path, "rb") as f:
f.seek(start)
remaining = end - start + 1
chunk_size = 64 * 1024
while remaining > 0:
to_read = chunk_size if remaining > chunk_size else remaining
data = f.read(to_read)
if not data:
break
try:
queue.put_nowait(data)
except Exception:
break
remaining -= len(data)
try:
queue.put_nowait(None)
except Exception:
pass
finally:
try:
sftp.close()
except Exception:
pass
async def agen():
worker_fut = asyncio.to_thread(_worker)
try:
while True:
chunk = await queue.get()
if chunk is None:
break
yield chunk
finally:
try:
await worker_fut
except Exception:
pass
return StreamingResponse(agen(), status_code=status, headers=headers, media_type=content_type)
ADAPTER_TYPE = "sftp"
CONFIG_SCHEMA = [
{"key": "host", "label": "主机", "type": "string", "required": True, "placeholder": "sftp.example.com"},
{"key": "port", "label": "端口", "type": "number", "required": False, "default": 22},
{"key": "username", "label": "用户名", "type": "string", "required": True},
{"key": "password", "label": "密码", "type": "password", "required": True},
{"key": "root", "label": "根路径", "type": "string", "required": True, "placeholder": "/data"},
{"key": "timeout", "label": "超时(秒)", "type": "number", "required": False, "default": 15},
{"key": "allow_unknown_host", "label": "允许未知主机指纹", "type": "boolean", "required": False, "default": True},
]
def ADAPTER_FACTORY(rec: StorageAdapter):
return SFTPAdapter(rec)
+578
View File
@@ -0,0 +1,578 @@
from typing import List, Dict, Tuple, AsyncIterator
import asyncio
import base64
import io
import os
import struct
from models import StorageAdapter
from telethon import TelegramClient
from telethon.crypto import AuthKey
from telethon.sessions import StringSession
from telethon.tl import types
import socks
_SESSION_LOCKS: Dict[str, asyncio.Lock] = {}
def _get_session_lock(session_string: str) -> asyncio.Lock:
lock = _SESSION_LOCKS.get(session_string)
if lock is None:
lock = asyncio.Lock()
_SESSION_LOCKS[session_string] = lock
return lock
class _NamedFile:
def __init__(self, file_obj, name: str):
self._file = file_obj
self.name = name
def read(self, *args, **kwargs):
return self._file.read(*args, **kwargs)
def seek(self, *args, **kwargs):
return self._file.seek(*args, **kwargs)
def tell(self):
return self._file.tell()
def seekable(self):
return self._file.seekable()
def close(self):
return self._file.close()
def __getattr__(self, name):
return getattr(self._file, name)
# 适配器类型标识
ADAPTER_TYPE = "telegram"
# 适配器配置项定义
CONFIG_SCHEMA = [
{"key": "api_id", "label": "API ID", "type": "string", "required": True, "help_text": "从 my.telegram.org 获取"},
{"key": "api_hash", "label": "API Hash", "type": "password", "required": True, "help_text": "从 my.telegram.org 获取"},
{"key": "session_string", "label": "Session String", "type": "password", "required": True, "help_text": "通过 generate_session.py 生成"},
{"key": "chat_id", "label": "Chat ID", "type": "string", "required": True, "placeholder": "频道/群组的ID或用户名, 例如: -100123456789 或 'channel_username'"},
{"key": "proxy_protocol", "label": "代理协议", "type": "string", "required": False, "placeholder": "例如: socks5, http"},
{"key": "proxy_host", "label": "代理主机", "type": "string", "required": False, "placeholder": "例如: 127.0.0.1"},
{"key": "proxy_port", "label": "代理端口", "type": "number", "required": False, "placeholder": "例如: 1080"},
]
class TelegramAdapter:
"""Telegram 存储适配器 (使用用户 Session)"""
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config
self.api_id = int(cfg.get("api_id"))
self.api_hash = cfg.get("api_hash")
self.session_string = cfg.get("session_string")
self.chat_id_str = cfg.get("chat_id")
# 代理设置
self.proxy_protocol = cfg.get("proxy_protocol")
self.proxy_host = cfg.get("proxy_host")
self.proxy_port = cfg.get("proxy_port")
self.proxy = None
if self.proxy_protocol and self.proxy_host and self.proxy_port:
proto_map = {
"socks5": socks.SOCKS5,
"http": socks.HTTP,
}
proxy_type = proto_map.get(self.proxy_protocol.lower())
if proxy_type:
self.proxy = (proxy_type, self.proxy_host, int(self.proxy_port))
try:
self.chat_id = int(self.chat_id_str)
except (ValueError, TypeError):
self.chat_id = self.chat_id_str
if not all([self.api_id, self.api_hash, self.session_string, self.chat_id]):
raise ValueError("Telegram 适配器需要 api_id, api_hash, session_string 和 chat_id")
@staticmethod
def _parse_legacy_session_string(value: str) -> StringSession:
"""
兼容旧版 session_string 格式:
- version(1B char) + base64(data)
- data: dc_id(1B) + ip_len(2B) + ip(ASCII, ip_len bytes) + port(2B) + auth_key(256B)
"""
s = (value or "").strip()
if not s:
raise ValueError("session_string 为空")
body = s[1:] if s.startswith("1") else s
raw = base64.urlsafe_b64decode(body)
if len(raw) < 1 + 2 + 2 + 256:
raise ValueError("legacy session 数据长度不足")
dc_id = raw[0]
ip_len = struct.unpack(">H", raw[1:3])[0]
expected_len = 1 + 2 + ip_len + 2 + 256
if len(raw) != expected_len:
raise ValueError("legacy session 数据长度不匹配")
ip_start = 3
ip_end = ip_start + ip_len
ip = raw[ip_start:ip_end].decode("utf-8")
port = struct.unpack(">H", raw[ip_end : ip_end + 2])[0]
key = raw[ip_end + 2 : ip_end + 2 + 256]
sess = StringSession()
sess.set_dc(dc_id, ip, port)
sess.auth_key = AuthKey(key)
return sess
@staticmethod
def _pick_photo_thumb(thumbs: list | None):
if not thumbs:
return None
cached = []
others = []
for t in thumbs:
if isinstance(t, (types.PhotoCachedSize, types.PhotoStrippedSize)):
cached.append(t)
elif isinstance(t, (types.PhotoSize, types.PhotoSizeProgressive)):
if not isinstance(t, types.PhotoSizeEmpty):
others.append(t)
if cached:
cached.sort(key=lambda x: len(getattr(x, "bytes", b"") or b""))
return cached[-1]
if others:
def _sz(x):
if isinstance(x, types.PhotoSizeProgressive):
return max(x.sizes or [0])
return int(getattr(x, "size", 0) or 0)
others.sort(key=_sz)
return others[-1]
return None
def _build_session(self) -> StringSession:
s = (self.session_string or "").strip()
if not s:
raise ValueError("Telegram 适配器 session_string 为空")
try:
return StringSession(s)
except Exception:
pass
# 少数工具可能去掉了 version 前缀,这里做一次兼容
if not s.startswith("1"):
try:
return StringSession("1" + s)
except Exception:
pass
try:
return self._parse_legacy_session_string(s)
except Exception as exc:
raise ValueError("Telegram session_string 无效,请使用 Telethon StringSession 重新生成") from exc
def _get_client(self) -> TelegramClient:
"""创建一个新的 TelegramClient 实例"""
return TelegramClient(self._build_session(), self.api_id, self.api_hash, proxy=self.proxy)
def get_effective_root(self, sub_path: str | None) -> str:
return ""
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
if rel:
return [], 0
client = self._get_client()
entries = []
try:
await client.connect()
messages = await client.get_messages(self.chat_id, limit=200)
for message in messages:
if not message:
continue
media = message.document or message.video or message.photo
if not media:
continue
file_meta = message.file
if not file_meta:
continue
filename = file_meta.name
if not filename:
if message.text and '.' in message.text and len(message.text) < 256 and '\n' not in message.text:
filename = message.text
else:
filename = f"unknown_{message.id}"
size = file_meta.size
if size is None:
# 兼容缺失 size 的情况
if hasattr(media, "size") and media.size is not None:
size = media.size
elif message.photo and getattr(message.photo, "sizes", None):
photo_size = message.photo.sizes[-1]
size = getattr(photo_size, "size", 0) or 0
else:
size = 0
entries.append({
"name": f"{message.id}_{filename}",
"is_dir": False,
"size": size,
"mtime": int(message.date.timestamp()),
"type": "file",
})
finally:
if client.is_connected():
await client.disconnect()
# 排序
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
key = (not item["is_dir"],)
sort_field = sort_by.lower()
if sort_field == "name":
key += (item["name"].lower(),)
elif sort_field == "size":
key += (item["size"],)
elif sort_field == "mtime":
key += (item["mtime"],)
else:
key += (item["name"].lower(),)
return key
entries.sort(key=get_sort_key, reverse=reverse)
total_count = len(entries)
# 分页
start_idx = (page_num - 1) * page_size
end_idx = start_idx + page_size
page_entries = entries[start_idx:end_idx]
return page_entries, total_count
async def read_file(self, root: str, rel: str) -> bytes:
try:
message_id_str, _ = rel.split('_', 1)
message_id = int(message_id_str)
except (ValueError, IndexError):
raise FileNotFoundError(f"无效的文件路径格式: {rel}")
client = self._get_client()
try:
await client.connect()
message = await client.get_messages(self.chat_id, ids=message_id)
if not message or not (message.document or message.video or message.photo):
raise FileNotFoundError(f"在频道 {self.chat_id} 中未找到消息ID为 {message_id} 的文件")
file_bytes = await client.download_media(message, file=bytes)
return file_bytes
finally:
if client.is_connected():
await client.disconnect()
async def write_file(self, root: str, rel: str, data: bytes):
"""将字节数据作为文件上传"""
client = self._get_client()
file_like = io.BytesIO(data)
file_like.name = os.path.basename(rel) or "file"
try:
await client.connect()
sent = await client.send_file(self.chat_id, file_like, caption=file_like.name)
message = sent[0] if isinstance(sent, list) and sent else sent
actual_rel = rel
if message:
stored_name = file_like.name
file_meta = getattr(message, "file", None)
if file_meta and getattr(file_meta, "name", None):
stored_name = file_meta.name
if getattr(message, "id", None) is not None:
actual_rel = f"{message.id}_{stored_name}"
return {"rel": actual_rel, "size": len(data)}
finally:
if client.is_connected():
await client.disconnect()
async def write_upload_file(self, root: str, rel: str, file_obj, filename: str | None, file_size: int | None = None, content_type: str | None = None):
client = self._get_client()
name = filename or os.path.basename(rel) or "file"
file_like = _NamedFile(file_obj, name)
try:
await client.connect()
sent = await client.send_file(
self.chat_id,
file_like,
caption=file_like.name,
file_size=file_size,
mime_type=content_type,
)
message = sent[0] if isinstance(sent, list) and sent else sent
actual_rel = rel
size = file_size or 0
if message:
stored_name = file_like.name
file_meta = getattr(message, "file", None)
if file_meta and getattr(file_meta, "name", None):
stored_name = file_meta.name
if getattr(message, "id", None) is not None:
actual_rel = f"{message.id}_{stored_name}"
if file_meta and getattr(file_meta, "size", None):
size = int(file_meta.size)
return {"rel": actual_rel, "size": size}
finally:
if client.is_connected():
await client.disconnect()
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
"""以流式方式上传文件"""
client = self._get_client()
filename = os.path.basename(rel) or "file"
import tempfile
suffix = os.path.splitext(filename)[1]
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tf:
temp_path = tf.name
total_size = 0
try:
with open(temp_path, "wb") as f:
async for chunk in data_iter:
if chunk:
f.write(chunk)
total_size += len(chunk)
await client.connect()
sent = await client.send_file(self.chat_id, temp_path, caption=filename)
message = sent[0] if isinstance(sent, list) and sent else sent
actual_rel = rel
if message:
stored_name = filename
file_meta = getattr(message, "file", None)
if file_meta and getattr(file_meta, "name", None):
stored_name = file_meta.name
if getattr(message, "id", None) is not None:
actual_rel = f"{message.id}_{stored_name}"
finally:
if os.path.exists(temp_path):
os.remove(temp_path)
if client.is_connected():
await client.disconnect()
return {"rel": actual_rel, "size": total_size}
async def mkdir(self, root: str, rel: str):
raise NotImplementedError("Telegram 适配器不支持创建目录。")
async def get_thumbnail(self, root: str, rel: str, size: str = "medium"):
try:
message_id_str, _ = rel.split('_', 1)
message_id = int(message_id_str)
except (ValueError, IndexError):
return None
client = self._get_client()
try:
await client.connect()
message = await client.get_messages(self.chat_id, ids=message_id)
if not message:
return None
doc = message.document or message.video
thumbs = None
if doc and getattr(doc, "thumbs", None):
thumbs = list(doc.thumbs or [])
elif message.photo and getattr(message.photo, "sizes", None):
thumbs = list(message.photo.sizes or [])
thumb = self._pick_photo_thumb(thumbs)
if not thumb:
return None
result = await client.download_media(message, bytes, thumb=thumb)
if isinstance(result, (bytes, bytearray)):
return bytes(result)
return None
except Exception:
return None
finally:
if client.is_connected():
await client.disconnect()
async def delete(self, root: str, rel: str):
"""删除一个文件 (即一条消息)"""
try:
message_id_str, _ = rel.split('_', 1)
message_id = int(message_id_str)
except (ValueError, IndexError):
raise FileNotFoundError(f"无效的文件路径格式,无法解析消息ID: {rel}")
client = self._get_client()
try:
await client.connect()
result = await client.delete_messages(self.chat_id, [message_id])
if not result or not result[0].pts:
raise FileNotFoundError(f"{self.chat_id} 中删除消息 {message_id} 失败,可能消息不存在或无权限")
finally:
if client.is_connected():
await client.disconnect()
async def move(self, root: str, src_rel: str, dst_rel: str):
raise NotImplementedError("Telegram 适配器不支持移动。")
async def rename(self, root: str, src_rel: str, dst_rel: str):
raise NotImplementedError("Telegram 适配器不支持重命名。")
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
raise NotImplementedError("Telegram 适配器不支持复制。")
async def stream_file(self, root: str, rel: str, range_header: str | None):
from fastapi.responses import StreamingResponse
from fastapi import HTTPException
try:
message_id_str, _ = rel.split('_', 1)
message_id = int(message_id_str)
except (ValueError, IndexError):
raise HTTPException(status_code=400, detail=f"无效的文件路径格式: {rel}")
client = self._get_client()
lock = _get_session_lock(self.session_string)
await lock.acquire()
try:
await client.connect()
message = await client.get_messages(self.chat_id, ids=message_id)
media = message.document or message.video or message.photo
if not message or not media:
raise FileNotFoundError(f"在频道 {self.chat_id} 中未找到消息ID为 {message_id} 的文件")
file_meta = message.file
file_size = file_meta.size if file_meta and file_meta.size is not None else None
if file_size is None:
if hasattr(media, "size") and media.size is not None:
file_size = media.size
elif message.photo and getattr(message.photo, "sizes", None):
photo_size = message.photo.sizes[-1]
file_size = getattr(photo_size, "size", 0) or 0
else:
file_size = 0
mime_type = None
if file_meta and getattr(file_meta, "mime_type", None):
mime_type = file_meta.mime_type
if not mime_type:
if hasattr(media, "mime_type") and media.mime_type:
mime_type = media.mime_type
elif message.photo:
mime_type = "image/jpeg"
else:
mime_type = "application/octet-stream"
start = 0
end = file_size - 1
status = 200
headers = {
"Accept-Ranges": "bytes",
"Content-Type": mime_type,
}
if range_header:
try:
range_val = range_header.strip().partition("=")[2]
s, _, e = range_val.partition("-")
start = int(s) if s else 0
end = int(e) if e else file_size - 1
if start >= file_size or end >= file_size or start > end:
raise HTTPException(status_code=416, detail="Requested Range Not Satisfiable")
status = 206
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
except ValueError:
raise HTTPException(status_code=400, detail="Invalid Range header")
async def iterator():
try:
limit = end - start + 1
downloaded = 0
async for chunk in client.iter_download(media, offset=start):
if downloaded + len(chunk) > limit:
yield chunk[:limit - downloaded]
break
yield chunk
downloaded += len(chunk)
if downloaded >= limit:
break
finally:
try:
if client.is_connected():
await client.disconnect()
finally:
lock.release()
return StreamingResponse(iterator(), status_code=status, headers=headers)
except HTTPException:
if client.is_connected():
await client.disconnect()
lock.release()
raise
except FileNotFoundError as e:
if client.is_connected():
await client.disconnect()
lock.release()
raise HTTPException(status_code=404, detail=str(e))
except Exception as e:
if client.is_connected():
await client.disconnect()
lock.release()
raise HTTPException(status_code=500, detail=f"Streaming failed: {str(e)}")
async def stat_file(self, root: str, rel: str):
try:
message_id_str, filename = rel.split('_', 1)
message_id = int(message_id_str)
except (ValueError, IndexError):
raise FileNotFoundError(f"无效的文件路径格式: {rel}")
client = self._get_client()
try:
await client.connect()
message = await client.get_messages(self.chat_id, ids=message_id)
media = message.document or message.video or message.photo
if not message or not media:
raise FileNotFoundError(f"在频道 {self.chat_id} 中未找到消息ID为 {message_id} 的文件")
file_meta = message.file
size = file_meta.size if file_meta and file_meta.size is not None else None
if size is None:
if hasattr(media, "size") and media.size is not None:
size = media.size
elif message.photo and getattr(message.photo, "sizes", None):
photo_size = message.photo.sizes[-1]
size = getattr(photo_size, "size", 0) or 0
else:
size = 0
return {
"name": rel,
"is_dir": False,
"size": size,
"mtime": int(message.date.timestamp()),
"type": "file",
}
finally:
if client.is_connected():
await client.disconnect()
def ADAPTER_FACTORY(rec: StorageAdapter) -> TelegramAdapter:
return TelegramAdapter(rec)
+492
View File
@@ -0,0 +1,492 @@
from typing import List, Dict, Optional, Tuple, AsyncIterator
import httpx
from urllib.parse import urljoin, quote
from urllib.parse import urlparse, unquote
import xml.etree.ElementTree as ET
from models import StorageAdapter
import mimetypes
import logging
from fastapi import HTTPException
from fastapi.responses import StreamingResponse, Response
NS = {"d": "DAV:"}
class WebDAVAdapter:
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config
self.base_url: str = cfg.get("base_url", "").rstrip('/') + '/'
if not self.base_url.startswith("http"):
raise ValueError("webdav requires base_url http/https")
self.username = cfg.get("username")
self.password = cfg.get("password")
self.timeout = cfg.get("timeout", 15)
def get_effective_root(self, sub_path: str | None) -> str:
base_url = self.record.config.get("base_url", "").rstrip('/') + '/'
if sub_path:
return base_url + sub_path.strip('/') + '/'
return base_url
def _client(self):
auth = (self.username, self.password) if self.username else None
return httpx.AsyncClient(auth=auth, timeout=self.timeout, follow_redirects=True)
def _build_url(self, rel: str):
rel = rel.strip('/')
return self.base_url if not rel else urljoin(self.base_url, quote(rel) + ('/' if rel.endswith('/') else ''))
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
raw_url = self._build_url(rel)
url = raw_url if raw_url.endswith('/') else raw_url + '/'
depth = "1"
body = """<?xml version="1.0" encoding="utf-8" ?>
<d:propfind xmlns:d="DAV:">
<d:prop>
<d:displayname />
<d:getcontentlength />
<d:getlastmodified />
<d:resourcetype />
</d:prop>
</d:propfind>"""
async with self._client() as client:
resp = await client.request("PROPFIND", url, data=body, headers={"Depth": depth})
resp.raise_for_status()
xml_text = resp.text
root_el = ET.fromstring(xml_text)
all_entries: List[Dict] = []
parsed_req = urlparse(url)
base_path = parsed_req.path
if not base_path.endswith('/'):
base_path += '/'
seen = set()
for resp_el in root_el.findall("d:response", NS):
href_el = resp_el.find("d:href", NS)
if href_el is None:
continue
href = (href_el.text or "")
parsed_href = urlparse(href)
href_path = parsed_href.path or ""
if not href_path.startswith(base_path):
continue
rel_path = href_path[len(base_path):].strip('/')
if rel_path == "":
continue
name = unquote(rel_path.split('/')[0]).rstrip('/')
if not name or name in seen:
continue
seen.add(name)
propstat = resp_el.find("d:propstat", NS)
if propstat is None:
continue
prop = propstat.find("d:prop", NS)
if prop is None:
continue
size_el = prop.find("d:getcontentlength", NS)
lm_el = prop.find("d:getlastmodified", NS)
rt_el = prop.find("d:resourcetype", NS)
is_dir = rt_el.find(
"d:collection", NS) is not None if rt_el is not None else href_path.endswith('/')
size = int(
size_el.text) if size_el is not None and size_el.text and size_el.text.isdigit() else 0
from email.utils import parsedate_to_datetime
mtime = 0
if lm_el is not None and lm_el.text:
try:
mtime = int(parsedate_to_datetime(lm_el.text).timestamp())
except Exception:
mtime = 0
all_entries.append({
"name": name,
"is_dir": is_dir,
"size": 0 if is_dir else size,
"mtime": mtime,
"type": "dir" if is_dir else "file",
})
# 排序所有条目
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
key = (not item["is_dir"],)
sort_field = sort_by.lower()
if sort_field == "name":
key += (item["name"].lower(),)
elif sort_field == "size":
key += (item["size"],)
elif sort_field == "mtime":
key += (item["mtime"],)
else:
key += (item["name"].lower(),)
return key
all_entries.sort(key=get_sort_key, reverse=reverse)
total_count = len(all_entries)
# 应用分页
start_idx = (page_num - 1) * page_size
end_idx = start_idx + page_size
page_entries = all_entries[start_idx:end_idx]
return page_entries, total_count
async def read_file(self, root: str, rel: str) -> bytes:
url = self._build_url(rel)
async with self._client() as client:
resp = await client.get(url)
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
return resp.content
async def write_file(self, root: str, rel: str, data: bytes):
url = self._build_url(rel)
async with self._client() as client:
resp = await client.put(url, content=data)
resp.raise_for_status()
async def mkdir(self, root: str, rel: str):
url = self._build_url(rel.rstrip('/') + '/')
async with self._client() as client:
resp = await client.request("MKCOL", url)
if resp.status_code not in (201, 405):
resp.raise_for_status()
async def delete(self, root: str, rel: str):
url = self._build_url(rel)
async with self._client() as client:
resp = await client.delete(url)
if resp.status_code not in (204, 200, 404):
resp.raise_for_status()
async def move(self, root: str, src_rel: str, dst_rel: str):
src_url = self._build_url(src_rel)
dst_url = self._build_url(dst_rel)
async with self._client() as client:
resp = await client.request("MOVE", src_url, headers={"Destination": dst_url})
resp.raise_for_status()
async def rename(self, root: str, src_rel: str, dst_rel: str):
src_url = self._build_url(src_rel)
dst_url = self._build_url(dst_rel)
async with self._client() as client:
resp = await client.request("MOVE", src_url, headers={"Destination": dst_url})
resp.raise_for_status()
async def get_file_size(self, root: str, rel: str) -> int:
"""获取文件大小"""
url = self._build_url(rel)
async with self._client() as client:
# 使用HEAD请求获取文件信息
resp = await client.head(url)
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
content_length = resp.headers.get('content-length')
if content_length:
return int(content_length)
# 如果HEAD不返回content-length,尝试PROPFIND
body = """<?xml version="1.0" encoding="utf-8" ?>
<d:propfind xmlns:d="DAV:">
<d:prop>
<d:getcontentlength />
</d:prop>
</d:propfind>"""
resp = await client.request("PROPFIND", url, data=body, headers={"Depth": "0"})
resp.raise_for_status()
root_el = ET.fromstring(resp.text)
for resp_el in root_el.findall("d:response", NS):
propstat = resp_el.find("d:propstat", NS)
if propstat is None:
continue
prop = propstat.find("d:prop", NS)
if prop is None:
continue
size_el = prop.find("d:getcontentlength", NS)
if size_el is not None and size_el.text and size_el.text.isdigit():
return int(size_el.text)
return 0
async def read_file_range(self, root: str, rel: str, start: int, end: Optional[int] = None) -> bytes:
"""读取文件的指定范围"""
url = self._build_url(rel)
# 构建Range头
if end is None:
range_header = f"bytes={start}-"
else:
range_header = f"bytes={start}-{end}"
async with self._client() as client:
resp = await client.get(url, headers={"Range": range_header})
if resp.status_code == 404:
raise FileNotFoundError(rel)
if resp.status_code not in (200, 206): # 206是Partial Content
resp.raise_for_status()
return resp.content
async def stream_file(self, root: str, rel: str, range_header: str | None):
url = self._build_url(rel)
mime, _ = mimetypes.guess_type(rel)
content_type = mime or "application/octet-stream"
logger = logging.getLogger(__name__)
timeout = self.timeout
auth = (self.username, self.password) if self.username else None
client_start = 0
client_end = None
status_code = 200
if range_header and range_header.startswith("bytes="):
status_code = 206
part = range_header.removeprefix("bytes=")
s, e = part.split("-", 1)
if s.strip():
client_start = int(s)
if e.strip():
client_end = int(e)
total_size = None
accept_ranges = False
async with httpx.AsyncClient(timeout=timeout, auth=auth, follow_redirects=True) as client:
try:
head_resp = await client.head(url)
if head_resp.status_code == 404:
raise HTTPException(404, detail="File not found")
if head_resp.status_code == 200:
cl = head_resp.headers.get("Content-Length")
if cl and cl.isdigit():
total_size = int(cl)
ar = head_resp.headers.get("Accept-Ranges", "").lower()
accept_ranges = "bytes" in ar
except HTTPException:
raise
except Exception as e:
logger.debug("HEAD failed %s err=%s", url, e)
if total_size is None and (client_end is None):
try:
probe_req = client.build_request("GET", url, headers={"Range": "bytes=0-0"})
probe_resp = await client.send(probe_req, stream=True)
if probe_resp.status_code in (200, 206):
cr = probe_resp.headers.get("Content-Range")
if cr and "/" in cr:
try:
total_size = int(cr.rsplit("/", 1)[1])
except Exception:
pass
await probe_resp.aclose()
except Exception as e:
logger.debug("Probe 0-0 failed %s err=%s", url, e)
if total_size is not None and client_end is None:
client_end = total_size - 1
if client_end is not None and client_end < client_start:
raise HTTPException(416, detail="Requested Range Not Satisfiable")
# 若客户端未请求范围且上游不支持 Range,直接透传
if status_code == 200 and (range_header is None) and not accept_ranges:
async with httpx.AsyncClient(timeout=timeout, auth=auth, follow_redirects=True) as client:
req = client.build_request("GET", url)
resp = await client.send(req, stream=True)
if resp.status_code == 404:
await resp.aclose()
raise HTTPException(404, detail="File not found")
upstream_ct = resp.headers.get("Content-Type", content_type)
async def passthrough():
try:
async for chunk in resp.aiter_bytes():
if chunk:
yield chunk
finally:
await resp.aclose()
return StreamingResponse(passthrough(), status_code=resp.status_code,
headers={"Accept-Ranges": "bytes",
"X-VFS-Remote-Status": str(resp.status_code)},
media_type=upstream_ct)
SEGMENT_SIZE = 5 * 1024 * 1024
MAX_RETRY_PER_SEG = 3
FIRST_BYTE_MAX_RETRY = 3
resp_headers = {
"Accept-Ranges": "bytes",
"Content-Type": content_type,
"X-VFS-Segmented": "1",
}
if status_code == 206 and total_size is not None:
resp_headers["Content-Range"] = f"bytes {client_start}-{client_end}/{total_size}"
async def segmented_body():
current = client_start
first_byte_sent = False
while True:
if client_end is not None and current > client_end:
break
seg_start = current
seg_end = (min(seg_start + SEGMENT_SIZE - 1, client_end)
if client_end is not None else seg_start + SEGMENT_SIZE - 1)
attempt = 0
ok = False
while attempt < MAX_RETRY_PER_SEG and not ok:
attempt += 1
headers_req = {"Range": f"bytes={seg_start}-{seg_end}"}
try:
async with httpx.AsyncClient(timeout=timeout, auth=auth, follow_redirects=True) as cseg:
req = cseg.build_request("GET", url, headers=headers_req)
rseg = await cseg.send(req, stream=True)
if rseg.status_code in (200, 206):
async for chunk in rseg.aiter_bytes():
if chunk:
first_byte_sent = True
yield chunk
await rseg.aclose()
ok = True
elif rseg.status_code == 404:
await rseg.aclose()
if not first_byte_sent:
raise HTTPException(404, detail="File not found")
return
else:
await rseg.aclose()
logger.warning("Segment unexpected status %s %s-%s %s", rel, seg_start, seg_end, rseg.status_code)
if not ok:
continue
except (httpx.ReadError, httpx.HTTPError, httpx.StreamError) as e:
if not first_byte_sent and attempt >= FIRST_BYTE_MAX_RETRY:
raise HTTPException(502, detail=f"Upstream error before first byte err={e}")
logger.warning("Segment error %s %s-%s attempt=%d err=%s", rel, seg_start, seg_end, attempt, e)
except Exception as e:
if not first_byte_sent:
raise
logger.error("Segment unexpected %s %s-%s attempt=%d err=%s", rel, seg_start, seg_end, attempt, e)
if not ok:
logger.error("Abort streaming %s at %s-%s", rel, seg_start, seg_end)
break
current = seg_end + 1
if client_end is None:
continue
if current > client_end:
break
return StreamingResponse(segmented_body(), status_code=status_code, headers=resp_headers, media_type=content_type)
async def stat_file(self, root: str, rel: str):
url = self._build_url(rel)
async with self._client() as client:
# PROPFIND 获取属性
body = """<?xml version="1.0" encoding="utf-8" ?>
<d:propfind xmlns:d="DAV:">
<d:prop>
<d:getcontentlength />
<d:getlastmodified />
<d:resourcetype />
</d:prop>
</d:propfind>"""
resp = await client.request("PROPFIND", url, data=body, headers={"Depth": "0"})
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
root_el = ET.fromstring(resp.text)
info = {
"name": rel.split("/")[-1],
"is_dir": False,
"size": None,
"mtime": None,
"type": "file",
"path": url,
}
for resp_el in root_el.findall("d:response", NS):
propstat = resp_el.find("d:propstat", NS)
if propstat is None:
continue
prop = propstat.find("d:prop", NS)
if prop is None:
continue
size_el = prop.find("d:getcontentlength", NS)
lm_el = prop.find("d:getlastmodified", NS)
rt_el = prop.find("d:resourcetype", NS)
is_dir = rt_el.find("d:collection", NS) is not None if rt_el is not None else False
info["is_dir"] = is_dir
info["type"] = "dir" if is_dir else "file"
if size_el is not None and size_el.text and size_el.text.isdigit():
info["size"] = int(size_el.text)
elif info["size"] is None:
info["size"] = 0
if lm_el is not None and lm_el.text:
from email.utils import parsedate_to_datetime
try:
info["mtime"] = int(parsedate_to_datetime(lm_el.text).timestamp())
except Exception:
info["mtime"] = 0
elif info["mtime"] is None:
info["mtime"] = 0
# exif信息
exif = None
if not info["is_dir"]:
mime, _ = mimetypes.guess_type(info["name"])
if mime and mime.startswith("image/"):
try:
resp_img = await client.get(url)
if resp_img.status_code == 200:
from PIL import Image
from io import BytesIO
img = Image.open(BytesIO(resp_img.content))
exif_data = img._getexif()
if exif_data:
exif = {str(k): str(v) for k, v in exif_data.items()}
except Exception:
exif = None
info["exif"] = exif
return info
async def exists(self, root: str, rel: str) -> bool:
url = self._build_url(rel)
async with self._client() as client:
try:
r = await client.head(url)
return r.status_code in (200, 204)
except Exception:
return False
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
url = self._build_url(rel)
async def agen():
async for chunk in data_iter:
if chunk:
yield chunk
async with self._client() as client:
resp = await client.put(url, content=agen())
resp.raise_for_status()
return True
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
src_url = self._build_url(src_rel)
dst_url = self._build_url(dst_rel)
headers = {
"Destination": dst_url,
"Overwrite": "T" if overwrite else "F"
}
async with self._client() as client:
resp = await client.request("COPY", src_url, headers=headers)
if resp.status_code == 412:
raise FileExistsError(dst_rel)
if resp.status_code == 404:
raise FileNotFoundError(src_rel)
resp.raise_for_status()
ADAPTER_TYPE = "webdav"
CONFIG_SCHEMA = [
{"key": "base_url", "label": "基础地址", "type": "string",
"required": True, "placeholder": "https://example.com/dav/"},
{"key": "username", "label": "用户名", "type": "string", "required": False},
{"key": "password", "label": "密码", "type": "password", "required": False},
{"key": "timeout",
"label": "超时(秒)", "type": "number", "required": False, "default": 15},
]
def ADAPTER_FACTORY(rec): return WebDAVAdapter(rec)
+157
View File
@@ -0,0 +1,157 @@
import inspect
import pkgutil
from importlib import import_module
from typing import Callable, Dict
from models import StorageAdapter
from .providers.base import BaseAdapter
AdapterFactory = Callable[[StorageAdapter], BaseAdapter]
TYPE_MAP: Dict[str, AdapterFactory] = {}
CONFIG_SCHEMAS: Dict[str, list] = {}
def normalize_adapter_type(value: str | None) -> str | None:
if value is None:
return None
normalized = str(value).strip().lower()
return normalized or None
def discover_adapters():
"""扫描 domain.adapters.providers 包, 自动注册适配器类型、工厂与配置 schema。"""
from . import providers as adapters_pkg
TYPE_MAP.clear()
CONFIG_SCHEMAS.clear()
for modinfo in pkgutil.iter_modules(adapters_pkg.__path__):
if modinfo.name.startswith("_"):
continue
full_name = f"{adapters_pkg.__name__}.{modinfo.name}"
try:
module = import_module(full_name)
except Exception:
continue
adapter_types = getattr(module, "ADAPTER_TYPES", None)
if isinstance(adapter_types, dict):
default_schema = getattr(module, "CONFIG_SCHEMA", None)
schema_map = getattr(module, "CONFIG_SCHEMA_MAP", None)
if not isinstance(schema_map, dict):
schema_map = None
for adapter_type, factory in adapter_types.items():
normalized_type = normalize_adapter_type(adapter_type)
if not normalized_type:
continue
if not callable(factory):
continue
TYPE_MAP[normalized_type] = factory
schema = schema_map.get(normalized_type) if schema_map else default_schema
if isinstance(schema, list):
CONFIG_SCHEMAS[normalized_type] = schema
continue
adapter_type = normalize_adapter_type(getattr(module, "ADAPTER_TYPE", None))
schema = getattr(module, "CONFIG_SCHEMA", None)
factory = getattr(module, "ADAPTER_FACTORY", None)
if not adapter_type:
continue
if factory is None:
for attr in module.__dict__.values():
if inspect.isclass(attr) and attr.__name__.endswith("Adapter"):
def _mk(cls=attr):
return lambda rec: cls(rec)
factory = _mk()
break
if not callable(factory):
continue
TYPE_MAP[adapter_type] = factory
if isinstance(schema, list):
CONFIG_SCHEMAS[adapter_type] = schema
def get_config_schemas() -> Dict[str, list]:
return CONFIG_SCHEMAS
def get_config_schema(adapter_type: str):
return CONFIG_SCHEMAS.get(adapter_type)
class RuntimeRegistry:
def __init__(self):
self._instances: Dict[int, BaseAdapter] = {}
async def refresh(self):
discover_adapters()
self._instances.clear()
adapters = await StorageAdapter.filter(enabled=True)
for rec in adapters:
normalized_type = normalize_adapter_type(rec.type)
if not normalized_type:
continue
if normalized_type != rec.type:
rec.type = normalized_type
try:
await rec.save(update_fields=["type"])
except Exception:
continue
factory = TYPE_MAP.get(normalized_type)
if not factory:
continue
try:
self._instances[rec.id] = factory(rec)
except Exception:
continue
def get(self, adapter_id: int) -> BaseAdapter | None:
return self._instances.get(adapter_id)
def snapshot(self) -> Dict[int, BaseAdapter]:
return dict(self._instances)
def remove(self, adapter_id: int):
"""从缓存中移除一个适配器实例"""
if adapter_id in self._instances:
del self._instances[adapter_id]
async def upsert(self, rec: StorageAdapter):
"""新增或更新一个适配器实例"""
if not rec.enabled:
self.remove(rec.id)
return
normalized_type = normalize_adapter_type(rec.type)
if not normalized_type:
self.remove(rec.id)
return
if normalized_type != rec.type:
rec.type = normalized_type
try:
await rec.save(update_fields=["type"])
except Exception:
pass
factory = TYPE_MAP.get(normalized_type)
if not factory:
discover_adapters()
factory = TYPE_MAP.get(normalized_type)
if not factory:
return
try:
instance = factory(rec)
self._instances[rec.id] = instance
except Exception:
self.remove(rec.id)
pass
runtime_registry = RuntimeRegistry()
discover_adapters()
+116
View File
@@ -0,0 +1,116 @@
from typing import Optional
from fastapi import HTTPException
from domain.auth import User
from .registry import (
get_config_schemas,
normalize_adapter_type,
runtime_registry,
)
from .types import AdapterCreate, AdapterOut
from models import StorageAdapter
class AdapterService:
@classmethod
def _validate_and_normalize_config(cls, adapter_type: str, cfg):
schemas = get_config_schemas()
adapter_type = normalize_adapter_type(adapter_type)
if not adapter_type:
raise HTTPException(400, detail="不支持的适配器类型")
if not isinstance(cfg, dict):
raise HTTPException(400, detail="config 必须是对象")
schema = schemas.get(adapter_type)
if not schema:
raise HTTPException(400, detail=f"不支持的适配器类型: {adapter_type}")
out = {}
missing = []
for f in schema:
k = f["key"]
if k in cfg and cfg[k] not in (None, ""):
out[k] = cfg[k]
elif "default" in f:
out[k] = f["default"]
elif f.get("required"):
missing.append(k)
if missing:
raise HTTPException(400, detail="缺少必填配置字段: " + ", ".join(missing))
if adapter_type in ("alist", "openlist"):
username = out.get("username")
password = out.get("password")
if (username and not password) or (password and not username):
raise HTTPException(400, detail="用户名和密码必须同时填写或同时留空")
return out
@classmethod
async def create_adapter(cls, data: AdapterCreate, current_user: Optional[User]):
norm_path = AdapterCreate.normalize_mount_path(data.path)
exists = await StorageAdapter.get_or_none(path=norm_path)
if exists:
raise HTTPException(400, detail="Mount path already exists")
adapter_fields = {
"name": data.name,
"type": data.type,
"config": cls._validate_and_normalize_config(data.type, data.config or {}),
"enabled": data.enabled,
"path": norm_path,
"sub_path": data.sub_path,
}
rec = await StorageAdapter.create(**adapter_fields)
await runtime_registry.upsert(rec)
return AdapterOut.model_validate(rec)
@classmethod
async def list_adapters(cls):
adapters = await StorageAdapter.all()
return [AdapterOut.model_validate(a) for a in adapters]
@classmethod
async def available_adapter_types(cls):
data = []
for adapter_type, fields in get_config_schemas().items():
data.append({
"type": adapter_type,
"config_schema": fields,
})
return data
@classmethod
async def get_adapter(cls, adapter_id: int):
rec = await StorageAdapter.get_or_none(id=adapter_id)
if not rec:
raise HTTPException(404, detail="Not found")
return AdapterOut.model_validate(rec)
@classmethod
async def update_adapter(cls, adapter_id: int, data: AdapterCreate, current_user: Optional[User]):
rec = await StorageAdapter.get_or_none(id=adapter_id)
if not rec:
raise HTTPException(404, detail="Not found")
norm_path = AdapterCreate.normalize_mount_path(data.path)
existing = await StorageAdapter.get_or_none(path=norm_path)
if existing and existing.id != adapter_id:
raise HTTPException(400, detail="Mount path already exists")
rec.name = data.name
rec.type = data.type
rec.config = cls._validate_and_normalize_config(data.type, data.config or {})
rec.enabled = data.enabled
rec.path = norm_path
rec.sub_path = data.sub_path
await rec.save()
await runtime_registry.upsert(rec)
return AdapterOut.model_validate(rec)
@classmethod
async def delete_adapter(cls, adapter_id: int, current_user: Optional[User]):
deleted = await StorageAdapter.filter(id=adapter_id).delete()
if not deleted:
raise HTTPException(404, detail="Not found")
runtime_registry.remove(adapter_id)
return {"deleted": True}
+50
View File
@@ -0,0 +1,50 @@
import re
from typing import Dict, Optional
from pydantic import BaseModel, Field, field_validator
class AdapterBase(BaseModel):
name: str
type: str = Field(pattern=r"^[a-z0-9_]+$")
config: Dict = Field(default_factory=dict)
enabled: bool = True
path: str = None
sub_path: Optional[str] = None
@field_validator("type", mode="before")
@classmethod
def _normalize_type(cls, v: str):
if not isinstance(v, str):
raise ValueError("type required")
normalized = v.strip().lower()
if not normalized:
raise ValueError("type required")
if not re.fullmatch(r"[a-z0-9_]+", normalized):
raise ValueError("type must be lowercase alphanumeric or underscore")
return normalized
class AdapterCreate(AdapterBase):
@staticmethod
def normalize_mount_path(p: str) -> str:
p = p.strip()
if not p.startswith('/'):
p = '/' + p
p = p.rstrip('/')
return p or '/'
@field_validator("path")
def _v_mount(cls, v: str):
if not v:
raise ValueError("mount_path required")
return cls.normalize_mount_path(v)
class AdapterOut(AdapterBase):
id: int
path: str = None
sub_path: Optional[str] = None
class Config:
from_attributes = True