mirror of
https://github.com/DrizzleTime/Foxel.git
synced 2026-09-04 23:29:00 +08:00
Initial commit
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
from .providers import BaseAdapter
|
||||
from .registry import (
|
||||
RuntimeRegistry,
|
||||
discover_adapters,
|
||||
get_config_schema,
|
||||
get_config_schemas,
|
||||
normalize_adapter_type,
|
||||
runtime_registry,
|
||||
)
|
||||
from .service import AdapterService
|
||||
from .types import AdapterCreate, AdapterOut
|
||||
|
||||
__all__ = [
|
||||
"BaseAdapter",
|
||||
"RuntimeRegistry",
|
||||
"discover_adapters",
|
||||
"get_config_schema",
|
||||
"get_config_schemas",
|
||||
"normalize_adapter_type",
|
||||
"runtime_registry",
|
||||
"AdapterService",
|
||||
"AdapterCreate",
|
||||
"AdapterOut",
|
||||
]
|
||||
@@ -0,0 +1,92 @@
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
|
||||
from api.response import success
|
||||
from domain.audit import AuditAction, audit
|
||||
from domain.auth import User, get_current_active_user
|
||||
from domain.permission import require_system_permission
|
||||
from domain.permission.types import AdapterPermission
|
||||
from .service import AdapterService
|
||||
from .types import AdapterCreate
|
||||
|
||||
router = APIRouter(prefix="/api/adapters", tags=["adapters"])
|
||||
|
||||
|
||||
@router.post("")
|
||||
@audit(
|
||||
action=AuditAction.CREATE,
|
||||
description="创建存储适配器",
|
||||
body_fields=["name", "type", "path", "sub_path", "enabled"],
|
||||
)
|
||||
@require_system_permission(AdapterPermission.CREATE)
|
||||
async def create_adapter(
|
||||
request: Request,
|
||||
data: AdapterCreate,
|
||||
current_user: Annotated[User, Depends(get_current_active_user)]
|
||||
):
|
||||
adapter = await AdapterService.create_adapter(data, current_user)
|
||||
return success(adapter)
|
||||
|
||||
|
||||
@router.get("")
|
||||
@audit(action=AuditAction.READ, description="获取适配器列表")
|
||||
@require_system_permission(AdapterPermission.LIST)
|
||||
async def list_adapters(
|
||||
request: Request,
|
||||
current_user: Annotated[User, Depends(get_current_active_user)]
|
||||
):
|
||||
adapters = await AdapterService.list_adapters()
|
||||
return success(adapters)
|
||||
|
||||
|
||||
@router.get("/available")
|
||||
@audit(action=AuditAction.READ, description="获取可用适配器类型")
|
||||
@require_system_permission(AdapterPermission.LIST)
|
||||
async def available_adapter_types(
|
||||
request: Request,
|
||||
current_user: Annotated[User, Depends(get_current_active_user)]
|
||||
):
|
||||
data = await AdapterService.available_adapter_types()
|
||||
return success(data)
|
||||
|
||||
|
||||
@router.get("/{adapter_id}")
|
||||
@audit(action=AuditAction.READ, description="获取适配器详情")
|
||||
@require_system_permission(AdapterPermission.LIST)
|
||||
async def get_adapter(
|
||||
request: Request,
|
||||
adapter_id: int,
|
||||
current_user: Annotated[User, Depends(get_current_active_user)]
|
||||
):
|
||||
adapter = await AdapterService.get_adapter(adapter_id)
|
||||
return success(adapter)
|
||||
|
||||
|
||||
@router.put("/{adapter_id}")
|
||||
@audit(
|
||||
action=AuditAction.UPDATE,
|
||||
description="更新存储适配器",
|
||||
body_fields=["name", "type", "path", "sub_path", "enabled"],
|
||||
)
|
||||
@require_system_permission(AdapterPermission.EDIT)
|
||||
async def update_adapter(
|
||||
request: Request,
|
||||
adapter_id: int,
|
||||
data: AdapterCreate,
|
||||
current_user: Annotated[User, Depends(get_current_active_user)]
|
||||
):
|
||||
adapter = await AdapterService.update_adapter(adapter_id, data, current_user)
|
||||
return success(adapter)
|
||||
|
||||
|
||||
@router.delete("/{adapter_id}")
|
||||
@audit(action=AuditAction.DELETE, description="删除存储适配器")
|
||||
@require_system_permission(AdapterPermission.DELETE)
|
||||
async def delete_adapter(
|
||||
request: Request,
|
||||
adapter_id: int,
|
||||
current_user: Annotated[User, Depends(get_current_active_user)]
|
||||
):
|
||||
result = await AdapterService.delete_adapter(adapter_id, current_user)
|
||||
return success(result)
|
||||
@@ -0,0 +1,3 @@
|
||||
from .base import BaseAdapter
|
||||
|
||||
__all__ = ["BaseAdapter"]
|
||||
@@ -0,0 +1,515 @@
|
||||
import asyncio
|
||||
import mimetypes
|
||||
import re
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, AsyncIterator, Dict, List, Tuple
|
||||
from urllib.parse import quote, urljoin
|
||||
|
||||
import httpx
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import Response, StreamingResponse
|
||||
|
||||
from models import StorageAdapter
|
||||
|
||||
|
||||
def _normalize_fs_path(path: str) -> str:
|
||||
path = (path or "").replace("\\", "/").strip()
|
||||
if not path or path == "/":
|
||||
return "/"
|
||||
if not path.startswith("/"):
|
||||
path = "/" + path
|
||||
path = re.sub(r"/{2,}", "/", path)
|
||||
if path != "/" and path.endswith("/"):
|
||||
path = path.rstrip("/")
|
||||
return path or "/"
|
||||
|
||||
|
||||
def _join_fs_path(base: str, rel: str) -> str:
|
||||
base = _normalize_fs_path(base)
|
||||
rel = (rel or "").replace("\\", "/").lstrip("/")
|
||||
if not rel:
|
||||
return base
|
||||
if base == "/":
|
||||
return "/" + rel
|
||||
return f"{base}/{rel}"
|
||||
|
||||
|
||||
def _split_parent_and_name(path: str) -> Tuple[str, str]:
|
||||
path = _normalize_fs_path(path)
|
||||
if path == "/":
|
||||
return "/", ""
|
||||
parent, _, name = path.rpartition("/")
|
||||
if not parent:
|
||||
parent = "/"
|
||||
return parent, name
|
||||
|
||||
|
||||
def _parse_iso_to_epoch(value: str | None) -> int:
|
||||
if not value:
|
||||
return 0
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return 0
|
||||
try:
|
||||
if text.endswith("Z"):
|
||||
text = text[:-1] + "+00:00"
|
||||
m = re.match(r"^(.*?)(\.\d+)([+-]\d\d:\d\d)?$", text)
|
||||
if m:
|
||||
head, frac, tz = m.group(1), m.group(2), m.group(3) or ""
|
||||
digits = frac[1:]
|
||||
if len(digits) > 6:
|
||||
frac = "." + digits[:6]
|
||||
text = head + frac + tz
|
||||
dt = datetime.fromisoformat(text)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return int(dt.timestamp())
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
class AListApiAdapterBase:
|
||||
def __init__(self, record: StorageAdapter, *, product_name: str):
|
||||
self.record = record
|
||||
self.product_name = product_name
|
||||
|
||||
cfg = record.config or {}
|
||||
self.base_url: str = str(cfg.get("base_url", "")).rstrip("/")
|
||||
if not self.base_url.startswith("http"):
|
||||
raise ValueError(f"{product_name} requires base_url http/https")
|
||||
self.username: str = str(cfg.get("username") or "")
|
||||
self.password: str = str(cfg.get("password") or "")
|
||||
if (self.username and not self.password) or (self.password and not self.username):
|
||||
raise ValueError(f"{product_name} requires both username and password")
|
||||
self.use_auth: bool = bool(self.username and self.password)
|
||||
|
||||
self.timeout: float = float(cfg.get("timeout", 30))
|
||||
self.root_path: str = _normalize_fs_path(str(cfg.get("root") or "/"))
|
||||
self.enable_redirect_307: bool = bool(cfg.get("enable_direct_download_307"))
|
||||
|
||||
self._token: str | None = None
|
||||
self._login_lock = asyncio.Lock()
|
||||
|
||||
def get_effective_root(self, sub_path: str | None) -> str:
|
||||
base = _normalize_fs_path(self.root_path)
|
||||
if sub_path:
|
||||
return _join_fs_path(base, sub_path)
|
||||
return base
|
||||
|
||||
async def _ensure_token(self) -> str:
|
||||
if not self.use_auth:
|
||||
return ""
|
||||
if self._token:
|
||||
return self._token
|
||||
async with self._login_lock:
|
||||
if self._token:
|
||||
return self._token
|
||||
self._token = await self._login()
|
||||
return self._token
|
||||
|
||||
async def _login(self) -> str:
|
||||
url = self.base_url + "/api/auth/login"
|
||||
body = {"username": self.username, "password": self.password}
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.post(url, json=body)
|
||||
resp.raise_for_status()
|
||||
payload = resp.json()
|
||||
if not isinstance(payload, dict):
|
||||
raise HTTPException(502, detail=f"{self.product_name} login: invalid response")
|
||||
code = payload.get("code")
|
||||
if code not in (0, 200):
|
||||
raise HTTPException(502, detail=f"{self.product_name} login failed: {payload.get('message')}")
|
||||
data = payload.get("data") or {}
|
||||
token = (data.get("token") if isinstance(data, dict) else None) or ""
|
||||
token = str(token).strip()
|
||||
if not token:
|
||||
raise HTTPException(502, detail=f"{self.product_name} login: missing token")
|
||||
return token
|
||||
|
||||
async def _api_json(
|
||||
self,
|
||||
method: str,
|
||||
endpoint: str,
|
||||
*,
|
||||
json: Dict[str, Any] | None = None,
|
||||
headers: Dict[str, str] | None = None,
|
||||
retry: bool = True,
|
||||
files: Any = None,
|
||||
) -> Any:
|
||||
token = await self._ensure_token()
|
||||
url = self.base_url + endpoint
|
||||
req_headers: Dict[str, str] = {}
|
||||
if token:
|
||||
req_headers["Authorization"] = token
|
||||
if headers:
|
||||
req_headers.update(headers)
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.request(method, url, json=json, headers=req_headers, files=files)
|
||||
if resp.status_code == 401 and retry and self.use_auth:
|
||||
self._token = None
|
||||
return await self._api_json(method, endpoint, json=json, headers=headers, retry=False, files=files)
|
||||
resp.raise_for_status()
|
||||
payload = resp.json()
|
||||
if not isinstance(payload, dict):
|
||||
raise HTTPException(502, detail=f"{self.product_name} api: invalid response")
|
||||
|
||||
code = payload.get("code")
|
||||
if code in (0, 200):
|
||||
return payload.get("data")
|
||||
if code in (401, 403) and retry and self.use_auth:
|
||||
self._token = None
|
||||
return await self._api_json(method, endpoint, json=json, headers=headers, retry=False, files=files)
|
||||
if code == 404:
|
||||
raise FileNotFoundError(json.get("path") if json else "")
|
||||
msg = payload.get("message") or payload.get("msg") or ""
|
||||
raise HTTPException(502, detail=f"{self.product_name} api error code={code} msg={msg}")
|
||||
|
||||
def _abs_url(self, url: str) -> str:
|
||||
u = (url or "").strip()
|
||||
if not u:
|
||||
return ""
|
||||
if u.startswith("http://") or u.startswith("https://"):
|
||||
return u
|
||||
return urljoin(self.base_url.rstrip("/") + "/", u.lstrip("/"))
|
||||
|
||||
async def _fs_list(self, path: str) -> Dict[str, Any]:
|
||||
body = {"path": path, "password": "", "page": 1, "per_page": 0, "refresh": False}
|
||||
data = await self._api_json("POST", "/api/fs/list", json=body)
|
||||
return data or {}
|
||||
|
||||
async def _fs_get(self, path: str) -> Dict[str, Any]:
|
||||
body = {"path": path, "password": "", "page": 1, "per_page": 0, "refresh": False}
|
||||
data = await self._api_json("POST", "/api/fs/get", json=body)
|
||||
return data or {}
|
||||
|
||||
async def list_dir(
|
||||
self,
|
||||
root: str,
|
||||
rel: str,
|
||||
page_num: int = 1,
|
||||
page_size: int = 50,
|
||||
sort_by: str = "name",
|
||||
sort_order: str = "asc",
|
||||
) -> Tuple[List[Dict], int]:
|
||||
path = _join_fs_path(root, rel)
|
||||
data = await self._fs_list(path)
|
||||
content = data.get("content") or []
|
||||
if not isinstance(content, list):
|
||||
raise HTTPException(502, detail=f"{self.product_name} list_dir: invalid content")
|
||||
|
||||
entries: List[Dict] = []
|
||||
for it in content:
|
||||
if not isinstance(it, dict):
|
||||
continue
|
||||
name = str(it.get("name") or "")
|
||||
if not name:
|
||||
continue
|
||||
is_dir = bool(it.get("is_dir"))
|
||||
size = int(it.get("size") or 0) if not is_dir else 0
|
||||
mtime = _parse_iso_to_epoch(it.get("modified"))
|
||||
entries.append(
|
||||
{
|
||||
"name": name,
|
||||
"is_dir": is_dir,
|
||||
"size": size,
|
||||
"mtime": mtime,
|
||||
"type": "dir" if is_dir else "file",
|
||||
}
|
||||
)
|
||||
|
||||
reverse = sort_order.lower() == "desc"
|
||||
|
||||
def get_sort_key(item: Dict) -> Tuple:
|
||||
key = (not item.get("is_dir"),)
|
||||
f = sort_by.lower()
|
||||
if f == "name":
|
||||
key += (str(item.get("name", "")).lower(),)
|
||||
elif f == "size":
|
||||
key += (int(item.get("size", 0)),)
|
||||
elif f == "mtime":
|
||||
key += (int(item.get("mtime", 0)),)
|
||||
else:
|
||||
key += (str(item.get("name", "")).lower(),)
|
||||
return key
|
||||
|
||||
entries.sort(key=get_sort_key, reverse=reverse)
|
||||
total = len(entries)
|
||||
start = (page_num - 1) * page_size
|
||||
end = start + page_size
|
||||
return entries[start:end], total
|
||||
|
||||
async def stat_file(self, root: str, rel: str):
|
||||
path = _join_fs_path(root, rel)
|
||||
data = await self._fs_get(path)
|
||||
if not data:
|
||||
raise FileNotFoundError(rel)
|
||||
is_dir = bool(data.get("is_dir"))
|
||||
name = str(data.get("name") or (rel.rstrip("/").split("/")[-1] if rel else ""))
|
||||
size = int(data.get("size") or 0) if not is_dir else 0
|
||||
mtime = _parse_iso_to_epoch(data.get("modified"))
|
||||
info = {
|
||||
"name": name,
|
||||
"is_dir": is_dir,
|
||||
"size": size,
|
||||
"mtime": mtime,
|
||||
"type": "dir" if is_dir else "file",
|
||||
"path": path,
|
||||
}
|
||||
return info
|
||||
|
||||
async def stat_path(self, root: str, rel: str):
|
||||
try:
|
||||
info = await self.stat_file(root, rel)
|
||||
return {"exists": True, "is_dir": bool(info.get("is_dir")), "path": info.get("path")}
|
||||
except FileNotFoundError:
|
||||
return {"exists": False, "is_dir": None, "path": _join_fs_path(root, rel)}
|
||||
|
||||
async def exists(self, root: str, rel: str) -> bool:
|
||||
try:
|
||||
await self.stat_file(root, rel)
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def get_direct_download_response(self, root: str, rel: str):
|
||||
if not self.enable_redirect_307:
|
||||
return None
|
||||
data = await self._fs_get(_join_fs_path(root, rel))
|
||||
if not data:
|
||||
raise FileNotFoundError(rel)
|
||||
if bool(data.get("is_dir")):
|
||||
raise IsADirectoryError(rel)
|
||||
raw_url = self._abs_url(str(data.get("raw_url") or ""))
|
||||
if not raw_url:
|
||||
return None
|
||||
return Response(status_code=307, headers={"Location": raw_url})
|
||||
|
||||
async def _get_raw_url_and_meta(self, root: str, rel: str) -> Tuple[str, int, str]:
|
||||
data = await self._fs_get(_join_fs_path(root, rel))
|
||||
if not data:
|
||||
raise FileNotFoundError(rel)
|
||||
if bool(data.get("is_dir")):
|
||||
raise IsADirectoryError(rel)
|
||||
raw_url = self._abs_url(str(data.get("raw_url") or ""))
|
||||
if not raw_url:
|
||||
raise HTTPException(502, detail=f"{self.product_name} missing raw_url")
|
||||
size = int(data.get("size") or 0)
|
||||
name = str(data.get("name") or "")
|
||||
return raw_url, size, name
|
||||
|
||||
async def read_file(self, root: str, rel: str) -> bytes:
|
||||
raw_url, _, _ = await self._get_raw_url_and_meta(root, rel)
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(raw_url)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
|
||||
async def stream_file(self, root: str, rel: str, range_header: str | None):
|
||||
raw_url, file_size, name = await self._get_raw_url_and_meta(root, rel)
|
||||
mime, _ = mimetypes.guess_type(name or rel)
|
||||
content_type = mime or "application/octet-stream"
|
||||
|
||||
start = 0
|
||||
end = max(file_size - 1, 0)
|
||||
status = 200
|
||||
headers = {
|
||||
"Accept-Ranges": "bytes",
|
||||
"Content-Type": content_type,
|
||||
}
|
||||
if file_size >= 0:
|
||||
headers["Content-Length"] = str(file_size)
|
||||
|
||||
if range_header and range_header.startswith("bytes="):
|
||||
try:
|
||||
part = range_header.removeprefix("bytes=")
|
||||
s, e = part.split("-", 1)
|
||||
if s.strip():
|
||||
start = int(s)
|
||||
if e.strip():
|
||||
end = int(e)
|
||||
if file_size and start >= file_size:
|
||||
raise HTTPException(416, detail="Requested Range Not Satisfiable")
|
||||
if file_size and end >= file_size:
|
||||
end = file_size - 1
|
||||
status = 206
|
||||
except ValueError:
|
||||
raise HTTPException(400, detail="Invalid Range header")
|
||||
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
|
||||
headers["Content-Length"] = str(end - start + 1)
|
||||
|
||||
async def agen():
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
req_headers = {"Range": f"bytes={start}-{end}"} if status == 206 else {}
|
||||
async with client.stream("GET", raw_url, headers=req_headers) as resp:
|
||||
resp.raise_for_status()
|
||||
async for chunk in resp.aiter_bytes():
|
||||
if chunk:
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(agen(), status_code=status, headers=headers, media_type=content_type)
|
||||
|
||||
async def _upload_file(self, full_path: str, file_path: Path) -> Any:
|
||||
token = await self._ensure_token()
|
||||
headers = {"File-Path": quote(full_path, safe="/")}
|
||||
if token:
|
||||
headers["Authorization"] = token
|
||||
with file_path.open("rb") as f:
|
||||
files = {"file": (file_path.name, f, "application/octet-stream")}
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.put(self.base_url + "/api/fs/form", headers=headers, files=files)
|
||||
resp.raise_for_status()
|
||||
payload = resp.json()
|
||||
if not isinstance(payload, dict):
|
||||
raise HTTPException(502, detail=f"{self.product_name} upload: invalid response")
|
||||
code = payload.get("code")
|
||||
if code not in (0, 200):
|
||||
msg = payload.get("message") or payload.get("msg") or ""
|
||||
raise HTTPException(502, detail=f"{self.product_name} upload failed: {msg}")
|
||||
return payload.get("data")
|
||||
|
||||
async def write_file(self, root: str, rel: str, data: bytes):
|
||||
full_path = _join_fs_path(root, rel)
|
||||
suffix = Path(rel).suffix
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tf:
|
||||
tf.write(data)
|
||||
tmp_path = Path(tf.name)
|
||||
try:
|
||||
await self._upload_file(full_path, tmp_path)
|
||||
finally:
|
||||
try:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def write_upload_file(self, root: str, rel: str, file_obj, filename: str | None, file_size: int | None = None, content_type: str | None = None):
|
||||
full_path = _join_fs_path(root, rel)
|
||||
token = await self._ensure_token()
|
||||
headers = {"File-Path": quote(full_path, safe="/")}
|
||||
if token:
|
||||
headers["Authorization"] = token
|
||||
name = filename or Path(rel).name or "file"
|
||||
mime = content_type or "application/octet-stream"
|
||||
files = {"file": (name, file_obj, mime)}
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.put(self.base_url + "/api/fs/form", headers=headers, files=files)
|
||||
resp.raise_for_status()
|
||||
payload = resp.json()
|
||||
if not isinstance(payload, dict):
|
||||
raise HTTPException(502, detail=f"{self.product_name} upload: invalid response")
|
||||
code = payload.get("code")
|
||||
if code not in (0, 200):
|
||||
msg = payload.get("message") or payload.get("msg") or ""
|
||||
raise HTTPException(502, detail=f"{self.product_name} upload failed: {msg}")
|
||||
data = payload.get("data")
|
||||
if isinstance(data, dict) and file_size is not None and "size" not in data:
|
||||
data["size"] = file_size
|
||||
return data
|
||||
|
||||
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
|
||||
full_path = _join_fs_path(root, rel)
|
||||
suffix = Path(rel).suffix
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tf:
|
||||
tmp_path = Path(tf.name)
|
||||
size = 0
|
||||
try:
|
||||
with tmp_path.open("wb") as f:
|
||||
async for chunk in data_iter:
|
||||
if not chunk:
|
||||
continue
|
||||
f.write(chunk)
|
||||
size += len(chunk)
|
||||
await self._upload_file(full_path, tmp_path)
|
||||
return size
|
||||
finally:
|
||||
try:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def mkdir(self, root: str, rel: str):
|
||||
path = _join_fs_path(root, rel)
|
||||
await self._api_json("POST", "/api/fs/mkdir", json={"path": path})
|
||||
|
||||
async def delete(self, root: str, rel: str):
|
||||
path = _join_fs_path(root, rel)
|
||||
parent, name = _split_parent_and_name(path)
|
||||
if not name:
|
||||
return
|
||||
await self._api_json("POST", "/api/fs/remove", json={"dir": parent, "names": [name]})
|
||||
|
||||
async def move(self, root: str, src_rel: str, dst_rel: str):
|
||||
src_path = _join_fs_path(root, src_rel)
|
||||
dst_path = _join_fs_path(root, dst_rel)
|
||||
src_dir, src_name = _split_parent_and_name(src_path)
|
||||
dst_dir, dst_name = _split_parent_and_name(dst_path)
|
||||
if not src_name or not dst_name:
|
||||
raise HTTPException(400, detail="Invalid move path")
|
||||
|
||||
if src_dir == dst_dir:
|
||||
if src_name == dst_name:
|
||||
return
|
||||
await self._api_json("POST", "/api/fs/rename", json={"path": src_path, "name": dst_name})
|
||||
return
|
||||
|
||||
await self._api_json("POST", "/api/fs/move", json={"src_dir": src_dir, "dst_dir": dst_dir, "names": [src_name]})
|
||||
if src_name != dst_name:
|
||||
await self._api_json("POST", "/api/fs/rename", json={"path": _join_fs_path(dst_dir, src_name), "name": dst_name})
|
||||
|
||||
async def rename(self, root: str, src_rel: str, dst_rel: str):
|
||||
await self.move(root, src_rel, dst_rel)
|
||||
|
||||
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
|
||||
src_path = _join_fs_path(root, src_rel)
|
||||
dst_path = _join_fs_path(root, dst_rel)
|
||||
src_dir, src_name = _split_parent_and_name(src_path)
|
||||
dst_dir, dst_name = _split_parent_and_name(dst_path)
|
||||
if not src_name or not dst_name:
|
||||
raise HTTPException(400, detail="Invalid copy path")
|
||||
|
||||
src_info = await self._fs_get(src_path)
|
||||
if not src_info:
|
||||
raise FileNotFoundError(src_rel)
|
||||
|
||||
if src_name != dst_name and not bool(src_info.get("is_dir")):
|
||||
raw_url, _, _ = await self._get_raw_url_and_meta(root, src_rel)
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
async with client.stream("GET", raw_url) as resp:
|
||||
resp.raise_for_status()
|
||||
|
||||
async def gen():
|
||||
async for chunk in resp.aiter_bytes():
|
||||
if chunk:
|
||||
yield chunk
|
||||
|
||||
await self.write_file_stream(root, dst_rel, gen())
|
||||
return
|
||||
|
||||
await self._api_json("POST", "/api/fs/copy", json={"src_dir": src_dir, "dst_dir": dst_dir, "names": [src_name]})
|
||||
if src_name != dst_name:
|
||||
await self._api_json("POST", "/api/fs/rename", json={"path": _join_fs_path(dst_dir, src_name), "name": dst_name})
|
||||
|
||||
|
||||
class AListAdapter(AListApiAdapterBase):
|
||||
def __init__(self, record: StorageAdapter):
|
||||
super().__init__(record, product_name="AList")
|
||||
|
||||
|
||||
class OpenListAdapter(AListApiAdapterBase):
|
||||
def __init__(self, record: StorageAdapter):
|
||||
super().__init__(record, product_name="OpenList")
|
||||
|
||||
|
||||
ADAPTER_TYPES = {"alist": AListAdapter, "openlist": OpenListAdapter}
|
||||
|
||||
CONFIG_SCHEMA = [
|
||||
{"key": "base_url", "label": "基础地址", "type": "string", "required": True, "placeholder": "http://127.0.0.1:5244"},
|
||||
{"key": "username", "label": "用户名", "type": "string", "required": False, "placeholder": "留空则匿名访问"},
|
||||
{"key": "password", "label": "密码", "type": "password", "required": False, "placeholder": "留空则匿名访问"},
|
||||
{"key": "root", "label": "根目录", "type": "string", "required": False, "default": "/"},
|
||||
{"key": "timeout", "label": "超时(秒)", "type": "number", "required": False, "default": 30},
|
||||
{"key": "enable_direct_download_307", "label": "启用 307 直链下载", "type": "boolean", "default": False},
|
||||
]
|
||||
@@ -0,0 +1,23 @@
|
||||
from typing import List, Dict, Protocol, runtime_checkable, Tuple, AsyncIterator
|
||||
from models import StorageAdapter
|
||||
|
||||
# 约定:任意新适配器模块需定义:
|
||||
# ADAPTER_TYPE: str
|
||||
# CONFIG_SCHEMA: List[Dict]
|
||||
# ADAPTER_FACTORY: Callable[[StorageAdapter], BaseAdapter] (可省略, 会自动寻找 *Adapter 类)
|
||||
|
||||
@runtime_checkable
|
||||
class BaseAdapter(Protocol):
|
||||
record: StorageAdapter
|
||||
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]: ...
|
||||
async def read_file(self, root: str, rel: str) -> bytes: ...
|
||||
async def write_file(self, root: str, rel: str, data: bytes): ...
|
||||
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]): ...
|
||||
async def mkdir(self, root: str, rel: str): ...
|
||||
async def delete(self, root: str, rel: str): ...
|
||||
async def move(self, root: str, src_rel: str, dst_rel: str): ...
|
||||
async def rename(self, root: str, src_rel: str, dst_rel: str): ...
|
||||
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False): ...
|
||||
async def stream_file(self, root: str, rel: str, range_header: str | None): ...
|
||||
async def stat_file(self, root: str, rel: str): ...
|
||||
def get_effective_root(self, sub_path: str | None) -> str: ...
|
||||
@@ -0,0 +1,471 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import mimetypes
|
||||
import re
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import AsyncIterator, Dict, List, Tuple
|
||||
|
||||
import httpx
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import Response, StreamingResponse
|
||||
|
||||
from models import StorageAdapter
|
||||
|
||||
DROPBOX_OAUTH_URL = "https://api.dropboxapi.com/oauth2/token"
|
||||
DROPBOX_API_URL = "https://api.dropboxapi.com/2"
|
||||
DROPBOX_CONTENT_URL = "https://content.dropboxapi.com/2"
|
||||
|
||||
|
||||
def _normalize_dbx_path(path: str | None) -> str:
|
||||
path = (path or "").replace("\\", "/").strip()
|
||||
if not path or path == "/":
|
||||
return ""
|
||||
if not path.startswith("/"):
|
||||
path = "/" + path
|
||||
path = re.sub(r"/{2,}", "/", path)
|
||||
if path.endswith("/"):
|
||||
path = path.rstrip("/")
|
||||
return path
|
||||
|
||||
|
||||
def _join_dbx_path(base: str, rel: str) -> str:
|
||||
base = _normalize_dbx_path(base)
|
||||
rel = (rel or "").replace("\\", "/").strip("/")
|
||||
if not rel:
|
||||
return base
|
||||
if not base:
|
||||
return "/" + rel
|
||||
return f"{base}/{rel}"
|
||||
|
||||
|
||||
def _parse_iso_to_epoch(value: str | None) -> int:
|
||||
if not value:
|
||||
return 0
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return 0
|
||||
try:
|
||||
if text.endswith("Z"):
|
||||
text = text[:-1] + "+00:00"
|
||||
dt = datetime.fromisoformat(text)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return int(dt.timestamp())
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
class DropboxAdapter:
|
||||
def __init__(self, record: StorageAdapter):
|
||||
self.record = record
|
||||
cfg = record.config or {}
|
||||
|
||||
self.app_key: str = str(cfg.get("app_key") or "").strip()
|
||||
self.app_secret: str = str(cfg.get("app_secret") or "").strip()
|
||||
self.refresh_token: str = str(cfg.get("refresh_token") or "").strip()
|
||||
self.root_path: str = _normalize_dbx_path(str(cfg.get("root") or "/"))
|
||||
self.enable_redirect_307: bool = bool(cfg.get("enable_direct_download_307"))
|
||||
self.timeout: float = float(cfg.get("timeout", 60))
|
||||
|
||||
if not (self.app_key and self.app_secret and self.refresh_token):
|
||||
raise ValueError("Dropbox 适配器需要 app_key, app_secret, refresh_token")
|
||||
|
||||
self._access_token: str | None = None
|
||||
self._token_expiry: datetime | None = None
|
||||
self._token_lock = asyncio.Lock()
|
||||
|
||||
def get_effective_root(self, sub_path: str | None) -> str:
|
||||
base = _normalize_dbx_path(self.root_path)
|
||||
if sub_path:
|
||||
return _join_dbx_path(base, sub_path)
|
||||
return base
|
||||
|
||||
async def _get_access_token(self) -> str:
|
||||
if self._access_token and self._token_expiry and datetime.now(timezone.utc) < self._token_expiry:
|
||||
return self._access_token
|
||||
|
||||
async with self._token_lock:
|
||||
if self._access_token and self._token_expiry and datetime.now(timezone.utc) < self._token_expiry:
|
||||
return self._access_token
|
||||
|
||||
basic = base64.b64encode(f"{self.app_key}:{self.app_secret}".encode("utf-8")).decode("ascii")
|
||||
headers = {"Authorization": f"Basic {basic}"}
|
||||
data = {"grant_type": "refresh_token", "refresh_token": self.refresh_token}
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
resp = await client.post(DROPBOX_OAUTH_URL, data=data, headers=headers)
|
||||
resp.raise_for_status()
|
||||
|
||||
payload = resp.json()
|
||||
token = str(payload.get("access_token") or "").strip()
|
||||
if not token:
|
||||
raise HTTPException(502, detail="Dropbox oauth: missing access_token")
|
||||
expires_in = int(payload.get("expires_in") or 3600)
|
||||
self._access_token = token
|
||||
self._token_expiry = datetime.now(timezone.utc) + timedelta(seconds=max(60, expires_in - 300))
|
||||
return token
|
||||
|
||||
async def _api_json(self, endpoint: str, body: Dict) -> httpx.Response:
|
||||
token = await self._get_access_token()
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
return await client.post(f"{DROPBOX_API_URL}{endpoint}", json=body, headers=headers)
|
||||
|
||||
async def _content_request(
|
||||
self,
|
||||
endpoint: str,
|
||||
api_arg: Dict,
|
||||
*,
|
||||
content: bytes | None = None,
|
||||
data_iter: AsyncIterator[bytes] | None = None,
|
||||
extra_headers: Dict[str, str] | None = None,
|
||||
) -> httpx.Response:
|
||||
token = await self._get_access_token()
|
||||
headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Dropbox-API-Arg": json.dumps(api_arg, separators=(",", ":"), ensure_ascii=False),
|
||||
}
|
||||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
|
||||
if data_iter is None:
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
return await client.post(f"{DROPBOX_CONTENT_URL}{endpoint}", headers=headers, content=content or b"")
|
||||
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
return await client.post(f"{DROPBOX_CONTENT_URL}{endpoint}", headers=headers, content=data_iter)
|
||||
|
||||
@staticmethod
|
||||
def _raise_dbx_error(resp: httpx.Response, *, rel: str):
|
||||
try:
|
||||
payload = resp.json()
|
||||
except Exception:
|
||||
payload = None
|
||||
summary = ""
|
||||
if isinstance(payload, dict):
|
||||
summary = str(payload.get("error_summary") or "")
|
||||
if "not_found" in summary:
|
||||
raise FileNotFoundError(rel)
|
||||
if "conflict" in summary or "already_exists" in summary:
|
||||
raise FileExistsError(rel)
|
||||
if "is_folder" in summary:
|
||||
raise IsADirectoryError(rel)
|
||||
if "not_folder" in summary:
|
||||
raise NotADirectoryError(rel)
|
||||
raise HTTPException(502, detail=f"Dropbox API error: {summary or resp.text}")
|
||||
|
||||
def _format_entry(self, entry: Dict) -> Dict:
|
||||
tag = entry.get(".tag")
|
||||
is_dir = tag == "folder"
|
||||
mtime = _parse_iso_to_epoch(entry.get("server_modified") if not is_dir else None)
|
||||
return {
|
||||
"name": entry.get("name") or "",
|
||||
"is_dir": is_dir,
|
||||
"size": 0 if is_dir else int(entry.get("size") or 0),
|
||||
"mtime": mtime,
|
||||
"type": "dir" if is_dir else "file",
|
||||
}
|
||||
|
||||
async def list_dir(
|
||||
self,
|
||||
root: str,
|
||||
rel: str,
|
||||
page_num: int = 1,
|
||||
page_size: int = 50,
|
||||
sort_by: str = "name",
|
||||
sort_order: str = "asc",
|
||||
) -> Tuple[List[Dict], int]:
|
||||
path = _join_dbx_path(root, rel)
|
||||
body = {"path": path, "recursive": False, "include_deleted": False, "limit": 2000}
|
||||
resp = await self._api_json("/files/list_folder", body)
|
||||
if resp.status_code == 409:
|
||||
try:
|
||||
payload = resp.json()
|
||||
except Exception:
|
||||
payload = None
|
||||
summary = str((payload or {}).get("error_summary") or "")
|
||||
if "not_found" in summary:
|
||||
return [], 0
|
||||
self._raise_dbx_error(resp, rel=rel)
|
||||
resp.raise_for_status()
|
||||
payload = resp.json()
|
||||
|
||||
all_entries: List[Dict] = []
|
||||
all_entries.extend(payload.get("entries") or [])
|
||||
cursor = payload.get("cursor")
|
||||
has_more = bool(payload.get("has_more"))
|
||||
while has_more and cursor:
|
||||
resp2 = await self._api_json("/files/list_folder/continue", {"cursor": cursor})
|
||||
resp2.raise_for_status()
|
||||
p2 = resp2.json()
|
||||
all_entries.extend(p2.get("entries") or [])
|
||||
cursor = p2.get("cursor")
|
||||
has_more = bool(p2.get("has_more"))
|
||||
|
||||
items = [self._format_entry(e) for e in all_entries if isinstance(e, dict)]
|
||||
|
||||
reverse = sort_order.lower() == "desc"
|
||||
|
||||
def get_sort_key(item):
|
||||
key = (not item["is_dir"],)
|
||||
f = sort_by.lower()
|
||||
if f == "name":
|
||||
key += (item["name"].lower(),)
|
||||
elif f == "size":
|
||||
key += (item["size"],)
|
||||
elif f == "mtime":
|
||||
key += (item["mtime"],)
|
||||
else:
|
||||
key += (item["name"].lower(),)
|
||||
return key
|
||||
|
||||
items.sort(key=get_sort_key, reverse=reverse)
|
||||
|
||||
total = len(items)
|
||||
start = (page_num - 1) * page_size
|
||||
end = start + page_size
|
||||
return items[start:end], total
|
||||
|
||||
async def stat_file(self, root: str, rel: str):
|
||||
path = _join_dbx_path(root, rel)
|
||||
resp = await self._api_json("/files/get_metadata", {"path": path, "include_deleted": False})
|
||||
if resp.status_code == 409:
|
||||
self._raise_dbx_error(resp, rel=rel)
|
||||
resp.raise_for_status()
|
||||
meta = resp.json()
|
||||
if not isinstance(meta, dict):
|
||||
raise HTTPException(502, detail="Dropbox metadata: invalid response")
|
||||
return self._format_entry(meta)
|
||||
|
||||
async def exists(self, root: str, rel: str) -> bool:
|
||||
try:
|
||||
await self.stat_file(root, rel)
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def read_file(self, root: str, rel: str) -> bytes:
|
||||
path = _join_dbx_path(root, rel)
|
||||
resp = await self._content_request("/files/download", {"path": path})
|
||||
if resp.status_code == 409:
|
||||
self._raise_dbx_error(resp, rel=rel)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
|
||||
async def write_file(self, root: str, rel: str, data: bytes):
|
||||
path = _join_dbx_path(root, rel)
|
||||
arg = {
|
||||
"path": path,
|
||||
"mode": "overwrite",
|
||||
"autorename": False,
|
||||
"mute": False,
|
||||
"strict_conflict": False,
|
||||
}
|
||||
resp = await self._content_request(
|
||||
"/files/upload",
|
||||
arg,
|
||||
content=data,
|
||||
extra_headers={"Content-Type": "application/octet-stream"},
|
||||
)
|
||||
if resp.status_code == 409:
|
||||
self._raise_dbx_error(resp, rel=rel)
|
||||
resp.raise_for_status()
|
||||
return True
|
||||
|
||||
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
|
||||
path = _join_dbx_path(root, rel)
|
||||
|
||||
size = 0
|
||||
session_id: str | None = None
|
||||
offset = 0
|
||||
|
||||
async for chunk in data_iter:
|
||||
if not chunk:
|
||||
continue
|
||||
if session_id is None:
|
||||
resp = await self._content_request(
|
||||
"/files/upload_session_start",
|
||||
{"close": False},
|
||||
content=chunk,
|
||||
extra_headers={"Content-Type": "application/octet-stream"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
payload = resp.json()
|
||||
session_id = str(payload.get("session_id") or "")
|
||||
if not session_id:
|
||||
raise HTTPException(502, detail="Dropbox upload_session_start: missing session_id")
|
||||
offset += len(chunk)
|
||||
size += len(chunk)
|
||||
continue
|
||||
|
||||
arg = {"cursor": {"session_id": session_id, "offset": offset}, "close": False}
|
||||
resp = await self._content_request(
|
||||
"/files/upload_session_append_v2",
|
||||
arg,
|
||||
content=chunk,
|
||||
extra_headers={"Content-Type": "application/octet-stream"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
offset += len(chunk)
|
||||
size += len(chunk)
|
||||
|
||||
if session_id is None:
|
||||
await self.write_file(root, rel, b"")
|
||||
return 0
|
||||
|
||||
finish_arg = {
|
||||
"cursor": {"session_id": session_id, "offset": offset},
|
||||
"commit": {
|
||||
"path": path,
|
||||
"mode": "overwrite",
|
||||
"autorename": False,
|
||||
"mute": False,
|
||||
"strict_conflict": False,
|
||||
},
|
||||
}
|
||||
resp = await self._content_request(
|
||||
"/files/upload_session_finish",
|
||||
finish_arg,
|
||||
content=b"",
|
||||
extra_headers={"Content-Type": "application/octet-stream"},
|
||||
)
|
||||
if resp.status_code == 409:
|
||||
self._raise_dbx_error(resp, rel=rel)
|
||||
resp.raise_for_status()
|
||||
return size
|
||||
|
||||
async def mkdir(self, root: str, rel: str):
|
||||
path = _join_dbx_path(root, rel)
|
||||
resp = await self._api_json("/files/create_folder_v2", {"path": path, "autorename": False})
|
||||
if resp.status_code == 409:
|
||||
self._raise_dbx_error(resp, rel=rel)
|
||||
resp.raise_for_status()
|
||||
return True
|
||||
|
||||
async def delete(self, root: str, rel: str):
|
||||
path = _join_dbx_path(root, rel)
|
||||
resp = await self._api_json("/files/delete_v2", {"path": path})
|
||||
if resp.status_code == 409:
|
||||
try:
|
||||
payload = resp.json()
|
||||
except Exception:
|
||||
payload = None
|
||||
summary = str((payload or {}).get("error_summary") or "")
|
||||
if "not_found" in summary:
|
||||
return
|
||||
self._raise_dbx_error(resp, rel=rel)
|
||||
resp.raise_for_status()
|
||||
return True
|
||||
|
||||
async def move(self, root: str, src_rel: str, dst_rel: str):
|
||||
src = _join_dbx_path(root, src_rel)
|
||||
dst = _join_dbx_path(root, dst_rel)
|
||||
resp = await self._api_json(
|
||||
"/files/move_v2",
|
||||
{"from_path": src, "to_path": dst, "autorename": False, "allow_shared_folder": True},
|
||||
)
|
||||
if resp.status_code == 409:
|
||||
self._raise_dbx_error(resp, rel=src_rel)
|
||||
resp.raise_for_status()
|
||||
return True
|
||||
|
||||
async def rename(self, root: str, src_rel: str, dst_rel: str):
|
||||
return await self.move(root, src_rel, dst_rel)
|
||||
|
||||
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
|
||||
src = _join_dbx_path(root, src_rel)
|
||||
dst = _join_dbx_path(root, dst_rel)
|
||||
resp = await self._api_json(
|
||||
"/files/copy_v2",
|
||||
{"from_path": src, "to_path": dst, "autorename": False, "allow_shared_folder": True},
|
||||
)
|
||||
if resp.status_code == 409:
|
||||
self._raise_dbx_error(resp, rel=dst_rel if overwrite else dst_rel)
|
||||
resp.raise_for_status()
|
||||
return True
|
||||
|
||||
async def get_direct_download_response(self, root: str, rel: str):
|
||||
if not self.enable_redirect_307:
|
||||
return None
|
||||
|
||||
path = _join_dbx_path(root, rel)
|
||||
resp = await self._api_json("/files/get_temporary_link", {"path": path})
|
||||
if resp.status_code == 409:
|
||||
self._raise_dbx_error(resp, rel=rel)
|
||||
resp.raise_for_status()
|
||||
payload = resp.json()
|
||||
link = (payload.get("link") if isinstance(payload, dict) else None) or ""
|
||||
link = str(link).strip()
|
||||
if not link:
|
||||
return None
|
||||
return Response(status_code=307, headers={"Location": link})
|
||||
|
||||
async def stream_file(self, root: str, rel: str, range_header: str | None):
|
||||
path = _join_dbx_path(root, rel)
|
||||
token = await self._get_access_token()
|
||||
headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Dropbox-API-Arg": json.dumps({"path": path}, separators=(",", ":"), ensure_ascii=False),
|
||||
}
|
||||
if range_header:
|
||||
headers["Range"] = range_header
|
||||
|
||||
client = httpx.AsyncClient(timeout=None)
|
||||
stream_cm = client.stream("POST", f"{DROPBOX_CONTENT_URL}/files/download", headers=headers)
|
||||
try:
|
||||
resp = await stream_cm.__aenter__()
|
||||
except Exception:
|
||||
await client.aclose()
|
||||
raise
|
||||
|
||||
if resp.status_code == 409:
|
||||
try:
|
||||
content = await resp.aread()
|
||||
_ = content
|
||||
finally:
|
||||
await stream_cm.__aexit__(None, None, None)
|
||||
await client.aclose()
|
||||
self._raise_dbx_error(resp, rel=rel)
|
||||
|
||||
if resp.status_code >= 400:
|
||||
try:
|
||||
await resp.aread()
|
||||
finally:
|
||||
await stream_cm.__aexit__(None, None, None)
|
||||
await client.aclose()
|
||||
resp.raise_for_status()
|
||||
|
||||
content_type = resp.headers.get("Content-Type") or (mimetypes.guess_type(rel)[0] or "application/octet-stream")
|
||||
out_headers = {}
|
||||
for key in ("Accept-Ranges", "Content-Range", "Content-Length"):
|
||||
value = resp.headers.get(key)
|
||||
if value:
|
||||
out_headers[key] = value
|
||||
|
||||
async def iterator():
|
||||
try:
|
||||
async for chunk in resp.aiter_bytes():
|
||||
if chunk:
|
||||
yield chunk
|
||||
finally:
|
||||
await stream_cm.__aexit__(None, None, None)
|
||||
await client.aclose()
|
||||
|
||||
return StreamingResponse(iterator(), status_code=resp.status_code, headers=out_headers, media_type=content_type)
|
||||
|
||||
|
||||
ADAPTER_TYPE = "dropbox"
|
||||
CONFIG_SCHEMA = [
|
||||
{"key": "app_key", "label": "App Key", "type": "string", "required": True},
|
||||
{"key": "app_secret", "label": "App Secret", "type": "password", "required": True},
|
||||
{"key": "refresh_token", "label": "Refresh Token", "type": "password", "required": True},
|
||||
{"key": "root", "label": "Root Path", "type": "string", "required": False, "default": "/", "placeholder": "/ or /Apps/Foxel"},
|
||||
{"key": "timeout", "label": "超时(秒)", "type": "number", "required": False, "default": 60},
|
||||
{"key": "enable_direct_download_307", "label": "Enable 307 redirect download", "type": "boolean", "default": False},
|
||||
]
|
||||
|
||||
|
||||
def ADAPTER_FACTORY(rec): return DropboxAdapter(rec)
|
||||
|
||||
@@ -0,0 +1,435 @@
|
||||
import asyncio
|
||||
import mimetypes
|
||||
import re
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any, AsyncIterator, Dict, List, Tuple
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from models import StorageAdapter
|
||||
|
||||
|
||||
def _normalize_fs_path(path: str) -> str:
|
||||
path = (path or "").replace("\\", "/").strip()
|
||||
if not path or path == "/":
|
||||
return "/"
|
||||
if not path.startswith("/"):
|
||||
path = "/" + path
|
||||
path = re.sub(r"/{2,}", "/", path)
|
||||
if path != "/" and path.endswith("/"):
|
||||
path = path.rstrip("/")
|
||||
return path or "/"
|
||||
|
||||
|
||||
def _join_fs_path(base: str, rel: str | None) -> str:
|
||||
base = _normalize_fs_path(base)
|
||||
rel_norm = (rel or "").replace("\\", "/").strip().lstrip("/")
|
||||
if not rel_norm:
|
||||
return base
|
||||
if base == "/":
|
||||
return "/" + rel_norm
|
||||
return f"{base}/{rel_norm}"
|
||||
|
||||
|
||||
def _unwrap_success(payload: Any, *, context: str) -> Any:
|
||||
if not isinstance(payload, dict):
|
||||
return payload
|
||||
if "data" not in payload:
|
||||
return payload
|
||||
code = payload.get("code")
|
||||
if code not in (None, 0, 200):
|
||||
msg = payload.get("msg") or payload.get("message") or ""
|
||||
raise HTTPException(502, detail=f"Foxel 上游错误({context}): {msg}")
|
||||
return payload.get("data")
|
||||
|
||||
|
||||
class FoxelAdapter:
|
||||
def __init__(self, record: StorageAdapter):
|
||||
self.record = record
|
||||
cfg = record.config or {}
|
||||
|
||||
self.base_url: str = str(cfg.get("base_url", "")).rstrip("/")
|
||||
if not self.base_url.startswith("http"):
|
||||
raise ValueError("foxel requires base_url http/https")
|
||||
|
||||
self.username: str = str(cfg.get("username") or "")
|
||||
self.password: str = str(cfg.get("password") or "")
|
||||
if not self.username or not self.password:
|
||||
raise ValueError("foxel requires username and password")
|
||||
|
||||
self.timeout: float = float(cfg.get("timeout", 15))
|
||||
self.root_path: str = _normalize_fs_path(str(cfg.get("root") or "/"))
|
||||
|
||||
self._token: str | None = None
|
||||
self._login_lock = asyncio.Lock()
|
||||
|
||||
def get_effective_root(self, sub_path: str | None) -> str:
|
||||
base = _normalize_fs_path(self.root_path)
|
||||
if sub_path:
|
||||
return _join_fs_path(base, sub_path)
|
||||
return base
|
||||
|
||||
async def _login(self) -> str:
|
||||
url = self.base_url + "/api/auth/login"
|
||||
body = {"username": self.username, "password": self.password}
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.post(url, data=body)
|
||||
resp.raise_for_status()
|
||||
payload = resp.json()
|
||||
if not isinstance(payload, dict):
|
||||
raise HTTPException(502, detail="Foxel 登录响应异常")
|
||||
token = payload.get("access_token")
|
||||
if not token:
|
||||
raise HTTPException(502, detail="Foxel 登录失败: 缺少 access_token")
|
||||
return str(token)
|
||||
|
||||
async def _ensure_token(self) -> str:
|
||||
if self._token:
|
||||
return self._token
|
||||
async with self._login_lock:
|
||||
if self._token:
|
||||
return self._token
|
||||
self._token = await self._login()
|
||||
return self._token
|
||||
|
||||
async def _request_json(self, method: str, path: str, *, params: dict | None = None, json: Any = None) -> Any:
|
||||
url = self.base_url + path
|
||||
for attempt in range(2):
|
||||
token = await self._ensure_token()
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.request(method, url, headers=headers, params=params, json=json)
|
||||
if resp.status_code == 401 and attempt == 0:
|
||||
self._token = None
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
raise HTTPException(502, detail="Foxel 上游请求失败")
|
||||
|
||||
@staticmethod
|
||||
def _encode_path(full_path: str) -> str:
|
||||
return quote(full_path.lstrip("/"), safe="/")
|
||||
|
||||
def _browse_path(self, full_path: str) -> str:
|
||||
full_path = _normalize_fs_path(full_path)
|
||||
if full_path == "/":
|
||||
return "/api/fs/"
|
||||
return "/api/fs/" + self._encode_path(full_path)
|
||||
|
||||
def _stat_path(self, full_path: str) -> str:
|
||||
full_path = _normalize_fs_path(full_path)
|
||||
if full_path == "/":
|
||||
return "/api/fs/stat/"
|
||||
return "/api/fs/stat/" + self._encode_path(full_path)
|
||||
|
||||
def _file_path(self, full_path: str) -> str:
|
||||
full_path = _normalize_fs_path(full_path)
|
||||
if full_path == "/":
|
||||
return "/api/fs/file/"
|
||||
return "/api/fs/file/" + self._encode_path(full_path)
|
||||
|
||||
def _stream_path(self, full_path: str) -> str:
|
||||
full_path = _normalize_fs_path(full_path)
|
||||
if full_path == "/":
|
||||
return "/api/fs/stream/"
|
||||
return "/api/fs/stream/" + self._encode_path(full_path)
|
||||
|
||||
async def list_dir(
|
||||
self,
|
||||
root: str,
|
||||
rel: str,
|
||||
page_num: int = 1,
|
||||
page_size: int = 50,
|
||||
sort_by: str = "name",
|
||||
sort_order: str = "asc",
|
||||
) -> Tuple[List[Dict], int]:
|
||||
rel = (rel or "").strip("/")
|
||||
full_path = _join_fs_path(root, rel)
|
||||
payload = await self._request_json(
|
||||
"GET",
|
||||
self._browse_path(full_path),
|
||||
params={
|
||||
"page": page_num,
|
||||
"page_size": page_size,
|
||||
"sort_by": sort_by,
|
||||
"sort_order": sort_order,
|
||||
},
|
||||
)
|
||||
data = _unwrap_success(payload, context="list_dir")
|
||||
if not isinstance(data, dict):
|
||||
raise HTTPException(502, detail="Foxel 浏览响应异常")
|
||||
entries = data.get("entries") or []
|
||||
pagination = data.get("pagination") or {}
|
||||
total = pagination.get("total")
|
||||
try:
|
||||
total_int = int(total) if total is not None else len(entries)
|
||||
except Exception:
|
||||
total_int = len(entries)
|
||||
if not isinstance(entries, list):
|
||||
entries = []
|
||||
return entries, total_int
|
||||
|
||||
async def stat_file(self, root: str, rel: str):
|
||||
rel = (rel or "").strip("/")
|
||||
full_path = _join_fs_path(root, rel)
|
||||
payload = await self._request_json("GET", self._stat_path(full_path))
|
||||
data = _unwrap_success(payload, context="stat_file")
|
||||
if not isinstance(data, dict):
|
||||
raise HTTPException(502, detail="Foxel stat 响应异常")
|
||||
return data
|
||||
|
||||
async def exists(self, root: str, rel: str) -> bool:
|
||||
rel = (rel or "").strip("/")
|
||||
full_path = _join_fs_path(root, rel)
|
||||
url = self.base_url + self._stat_path(full_path)
|
||||
for attempt in range(2):
|
||||
token = await self._ensure_token()
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=headers)
|
||||
if resp.status_code == 401 and attempt == 0:
|
||||
self._token = None
|
||||
continue
|
||||
return resp.status_code == 200
|
||||
return False
|
||||
|
||||
async def read_file(self, root: str, rel: str) -> bytes:
|
||||
rel = (rel or "").lstrip("/")
|
||||
full_path = _join_fs_path(root, rel)
|
||||
url = self.base_url + self._file_path(full_path)
|
||||
for attempt in range(2):
|
||||
token = await self._ensure_token()
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=headers)
|
||||
if resp.status_code == 401 and attempt == 0:
|
||||
self._token = None
|
||||
continue
|
||||
if resp.status_code == 404:
|
||||
raise FileNotFoundError(rel)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
raise HTTPException(502, detail="Foxel 读取失败")
|
||||
|
||||
async def _upload_file_path(self, full_path: str, file_path: Path) -> None:
|
||||
url = self.base_url + self._file_path(full_path)
|
||||
filename = Path(full_path).name or file_path.name
|
||||
for attempt in range(2):
|
||||
token = await self._ensure_token()
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
with file_path.open("rb") as f:
|
||||
files = {"file": (filename, f, "application/octet-stream")}
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.post(url, headers=headers, files=files)
|
||||
if resp.status_code == 401 and attempt == 0:
|
||||
self._token = None
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
return
|
||||
raise HTTPException(502, detail="Foxel 上传失败")
|
||||
|
||||
async def write_file(self, root: str, rel: str, data: bytes):
|
||||
rel = (rel or "").lstrip("/")
|
||||
full_path = _join_fs_path(root, rel)
|
||||
url = self.base_url + self._file_path(full_path)
|
||||
filename = Path(rel).name or "file"
|
||||
for attempt in range(2):
|
||||
token = await self._ensure_token()
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
files = {"file": (filename, data, "application/octet-stream")}
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.post(url, headers=headers, files=files)
|
||||
if resp.status_code == 401 and attempt == 0:
|
||||
self._token = None
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
return True
|
||||
raise HTTPException(502, detail="Foxel 写入失败")
|
||||
|
||||
async def write_upload_file(self, root: str, rel: str, file_obj, filename: str | None, file_size: int | None = None, content_type: str | None = None):
|
||||
rel = (rel or "").lstrip("/")
|
||||
full_path = _join_fs_path(root, rel)
|
||||
url = self.base_url + self._file_path(full_path)
|
||||
name = filename or Path(rel).name or "file"
|
||||
mime = content_type or "application/octet-stream"
|
||||
for attempt in range(2):
|
||||
try:
|
||||
if callable(getattr(file_obj, "seek", None)):
|
||||
file_obj.seek(0)
|
||||
except Exception:
|
||||
pass
|
||||
token = await self._ensure_token()
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
files = {"file": (name, file_obj, mime)}
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.post(url, headers=headers, files=files)
|
||||
if resp.status_code == 401 and attempt == 0:
|
||||
self._token = None
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
return {"size": file_size or 0}
|
||||
raise HTTPException(502, detail="Foxel 上传失败")
|
||||
|
||||
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
|
||||
rel = (rel or "").lstrip("/")
|
||||
full_path = _join_fs_path(root, rel)
|
||||
suffix = Path(rel).suffix
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tf:
|
||||
tmp_path = Path(tf.name)
|
||||
|
||||
size = 0
|
||||
try:
|
||||
with tmp_path.open("wb") as f:
|
||||
async for chunk in data_iter:
|
||||
if not chunk:
|
||||
continue
|
||||
f.write(chunk)
|
||||
size += len(chunk)
|
||||
await self._upload_file_path(full_path, tmp_path)
|
||||
return size
|
||||
finally:
|
||||
try:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def mkdir(self, root: str, rel: str):
|
||||
rel = (rel or "").strip("/")
|
||||
full_path = _join_fs_path(root, rel)
|
||||
payload = await self._request_json("POST", "/api/fs/mkdir", json={"path": full_path})
|
||||
_unwrap_success(payload, context="mkdir")
|
||||
return True
|
||||
|
||||
async def delete(self, root: str, rel: str):
|
||||
rel = (rel or "").strip("/")
|
||||
full_path = _join_fs_path(root, rel)
|
||||
url = self.base_url + self._browse_path(full_path)
|
||||
for attempt in range(2):
|
||||
token = await self._ensure_token()
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
|
||||
resp = await client.delete(url, headers=headers)
|
||||
if resp.status_code == 401 and attempt == 0:
|
||||
self._token = None
|
||||
continue
|
||||
if resp.status_code == 404:
|
||||
return
|
||||
resp.raise_for_status()
|
||||
return
|
||||
raise HTTPException(502, detail="Foxel 删除失败")
|
||||
|
||||
async def move(self, root: str, src_rel: str, dst_rel: str):
|
||||
src_path = _join_fs_path(root, (src_rel or "").lstrip("/"))
|
||||
dst_path = _join_fs_path(root, (dst_rel or "").lstrip("/"))
|
||||
payload = await self._request_json("POST", "/api/fs/move", json={"src": src_path, "dst": dst_path})
|
||||
_unwrap_success(payload, context="move")
|
||||
return True
|
||||
|
||||
async def rename(self, root: str, src_rel: str, dst_rel: str):
|
||||
src_path = _join_fs_path(root, (src_rel or "").lstrip("/"))
|
||||
dst_path = _join_fs_path(root, (dst_rel or "").lstrip("/"))
|
||||
payload = await self._request_json("POST", "/api/fs/rename", json={"src": src_path, "dst": dst_path})
|
||||
_unwrap_success(payload, context="rename")
|
||||
return True
|
||||
|
||||
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
|
||||
src_path = _join_fs_path(root, (src_rel or "").lstrip("/"))
|
||||
dst_path = _join_fs_path(root, (dst_rel or "").lstrip("/"))
|
||||
payload = await self._request_json(
|
||||
"POST",
|
||||
"/api/fs/copy",
|
||||
json={"src": src_path, "dst": dst_path},
|
||||
params={"overwrite": overwrite},
|
||||
)
|
||||
_unwrap_success(payload, context="copy")
|
||||
return True
|
||||
|
||||
async def stream_file(self, root: str, rel: str, range_header: str | None):
|
||||
rel = (rel or "").lstrip("/")
|
||||
full_path = _join_fs_path(root, rel)
|
||||
url = self.base_url + self._stream_path(full_path)
|
||||
|
||||
headers = {}
|
||||
if range_header:
|
||||
headers["Range"] = range_header
|
||||
|
||||
for attempt in range(2):
|
||||
token = await self._ensure_token()
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
client = httpx.AsyncClient(timeout=None, follow_redirects=True)
|
||||
stream_cm = client.stream("GET", url, headers=headers)
|
||||
try:
|
||||
resp = await stream_cm.__aenter__()
|
||||
except Exception:
|
||||
await client.aclose()
|
||||
raise
|
||||
|
||||
if resp.status_code == 401 and attempt == 0:
|
||||
try:
|
||||
await resp.aread()
|
||||
finally:
|
||||
await stream_cm.__aexit__(None, None, None)
|
||||
await client.aclose()
|
||||
self._token = None
|
||||
continue
|
||||
|
||||
if resp.status_code == 404:
|
||||
try:
|
||||
await resp.aread()
|
||||
finally:
|
||||
await stream_cm.__aexit__(None, None, None)
|
||||
await client.aclose()
|
||||
raise FileNotFoundError(rel)
|
||||
|
||||
if resp.status_code >= 400:
|
||||
try:
|
||||
await resp.aread()
|
||||
finally:
|
||||
await stream_cm.__aexit__(None, None, None)
|
||||
await client.aclose()
|
||||
resp.raise_for_status()
|
||||
|
||||
content_type = resp.headers.get("Content-Type") or (
|
||||
mimetypes.guess_type(rel)[0] or "application/octet-stream"
|
||||
)
|
||||
out_headers = {}
|
||||
for key in ("Accept-Ranges", "Content-Range", "Content-Length"):
|
||||
value = resp.headers.get(key)
|
||||
if value:
|
||||
out_headers[key] = value
|
||||
|
||||
async def iterator():
|
||||
try:
|
||||
async for chunk in resp.aiter_bytes():
|
||||
if chunk:
|
||||
yield chunk
|
||||
finally:
|
||||
await stream_cm.__aexit__(None, None, None)
|
||||
await client.aclose()
|
||||
|
||||
return StreamingResponse(
|
||||
iterator(),
|
||||
status_code=resp.status_code,
|
||||
headers=out_headers,
|
||||
media_type=content_type,
|
||||
)
|
||||
|
||||
raise HTTPException(502, detail="Foxel 流式读取失败")
|
||||
|
||||
|
||||
ADAPTER_TYPE = "foxel"
|
||||
CONFIG_SCHEMA = [
|
||||
{"key": "base_url", "label": "节点地址", "type": "string", "required": True, "placeholder": "http://127.0.0.1:8000"},
|
||||
{"key": "username", "label": "用户名", "type": "string", "required": True},
|
||||
{"key": "password", "label": "密码", "type": "password", "required": True},
|
||||
{"key": "root", "label": "远端根目录", "type": "string", "required": False, "default": "/", "placeholder": "/ 或 /drive"},
|
||||
{"key": "timeout", "label": "超时(秒)", "type": "number", "required": False, "default": 60},
|
||||
]
|
||||
|
||||
|
||||
def ADAPTER_FACTORY(rec: StorageAdapter):
|
||||
return FoxelAdapter(rec)
|
||||
@@ -0,0 +1,645 @@
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Dict, Tuple, AsyncIterator, Optional
|
||||
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
from ftplib import FTP, error_perm
|
||||
import mimetypes
|
||||
|
||||
from models import StorageAdapter
|
||||
|
||||
|
||||
def _join_remote(root: str, rel: str) -> str:
|
||||
root = (root or "/").rstrip("/") or "/"
|
||||
rel = (rel or "").lstrip("/")
|
||||
if not rel:
|
||||
return root
|
||||
return f"{root}/{rel}"
|
||||
|
||||
|
||||
def _parse_mlst_line(line: str) -> Dict[str, str]:
|
||||
out: Dict[str, str] = {}
|
||||
try:
|
||||
facts, _, name = line.partition(" ")
|
||||
for part in facts.split(";"):
|
||||
if not part or "=" not in part:
|
||||
continue
|
||||
k, v = part.split("=", 1)
|
||||
out[k.strip().lower()] = v.strip()
|
||||
if name:
|
||||
out["name"] = name.strip()
|
||||
except Exception:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def _parse_modify_to_epoch(mod: str) -> int:
|
||||
# Formats we may see: YYYYMMDDHHMMSS or YYYYMMDDHHMMSS(.sss)
|
||||
try:
|
||||
mod = mod.strip()
|
||||
mod = mod.split(".")[0]
|
||||
if len(mod) >= 14:
|
||||
y = int(mod[0:4])
|
||||
m = int(mod[4:6])
|
||||
d = int(mod[6:8])
|
||||
hh = int(mod[8:10])
|
||||
mm = int(mod[10:12])
|
||||
ss = int(mod[12:14])
|
||||
import datetime as _dt
|
||||
return int(_dt.datetime(y, m, d, hh, mm, ss, tzinfo=_dt.timezone.utc).timestamp())
|
||||
except Exception:
|
||||
return 0
|
||||
return 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Range:
|
||||
start: int
|
||||
end: Optional[int] # inclusive
|
||||
|
||||
|
||||
class FTPAdapter:
|
||||
def __init__(self, record: StorageAdapter):
|
||||
self.record = record
|
||||
cfg = record.config
|
||||
self.host: str = cfg.get("host")
|
||||
self.port: int = int(cfg.get("port", 21))
|
||||
self.username: Optional[str] = cfg.get("username")
|
||||
self.password: Optional[str] = cfg.get("password")
|
||||
self.passive: bool = bool(cfg.get("passive", True))
|
||||
self.timeout: int = int(cfg.get("timeout", 15))
|
||||
self.root_path: str = cfg.get("root", "/") or "/"
|
||||
|
||||
if not self.host:
|
||||
raise ValueError("FTP adapter requires 'host'")
|
||||
|
||||
def get_effective_root(self, sub_path: str | None) -> str:
|
||||
base = self.root_path.rstrip("/") or "/"
|
||||
if sub_path:
|
||||
return _join_remote(base, sub_path)
|
||||
return base
|
||||
|
||||
def _connect(self) -> FTP:
|
||||
ftp = FTP()
|
||||
ftp.connect(self.host, self.port, timeout=self.timeout)
|
||||
if self.username:
|
||||
ftp.login(self.username, self.password or "")
|
||||
else:
|
||||
ftp.login()
|
||||
ftp.set_pasv(self.passive)
|
||||
return ftp
|
||||
|
||||
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
|
||||
path = _join_remote(root, rel.strip('/'))
|
||||
|
||||
def _do_list() -> List[Dict]:
|
||||
ftp = self._connect()
|
||||
try:
|
||||
ftp.cwd(path)
|
||||
except error_perm as e:
|
||||
# path may be file
|
||||
ftp.quit()
|
||||
raise NotADirectoryError(rel) from e
|
||||
|
||||
entries: List[Dict] = []
|
||||
# Try MLSD first
|
||||
try:
|
||||
for name, facts in ftp.mlsd():
|
||||
if name in (".", ".."):
|
||||
continue
|
||||
is_dir = (facts.get("type") == "dir")
|
||||
size = int(facts.get("size") or 0)
|
||||
mtime = _parse_modify_to_epoch(facts.get("modify") or "")
|
||||
entries.append({
|
||||
"name": name,
|
||||
"is_dir": is_dir,
|
||||
"size": 0 if is_dir else size,
|
||||
"mtime": mtime,
|
||||
"type": "dir" if is_dir else "file",
|
||||
})
|
||||
ftp.quit()
|
||||
return entries
|
||||
except Exception:
|
||||
# Fallback to NLST + probing
|
||||
pass
|
||||
|
||||
names = []
|
||||
try:
|
||||
names = ftp.nlst()
|
||||
except Exception:
|
||||
ftp.quit()
|
||||
return []
|
||||
|
||||
for name in names:
|
||||
if name in (".", ".."):
|
||||
continue
|
||||
is_dir = False
|
||||
size = 0
|
||||
mtime = 0
|
||||
try:
|
||||
# If we can CWD, it's a directory
|
||||
ftp.cwd(_join_remote(path, name))
|
||||
ftp.cwd(path)
|
||||
is_dir = True
|
||||
except Exception:
|
||||
is_dir = False
|
||||
try:
|
||||
size = ftp.size(_join_remote(path, name)) or 0
|
||||
except Exception:
|
||||
size = 0
|
||||
try:
|
||||
mdtm = ftp.sendcmd("MDTM " + _join_remote(path, name))
|
||||
# Example: '213 20241012XXXXXX'
|
||||
if mdtm.startswith("213 "):
|
||||
mtime = _parse_modify_to_epoch(mdtm.split(" ", 1)[1])
|
||||
except Exception:
|
||||
pass
|
||||
entries.append({
|
||||
"name": name,
|
||||
"is_dir": is_dir,
|
||||
"size": 0 if is_dir else int(size or 0),
|
||||
"mtime": int(mtime or 0),
|
||||
"type": "dir" if is_dir else "file",
|
||||
})
|
||||
ftp.quit()
|
||||
return entries
|
||||
|
||||
entries = await asyncio.to_thread(_do_list)
|
||||
|
||||
reverse = sort_order.lower() == "desc"
|
||||
|
||||
def get_sort_key(item):
|
||||
key = (not item["is_dir"],)
|
||||
f = sort_by.lower()
|
||||
if f == "name":
|
||||
key += (item["name"].lower(),)
|
||||
elif f == "size":
|
||||
key += (item.get("size", 0),)
|
||||
elif f == "mtime":
|
||||
key += (item.get("mtime", 0),)
|
||||
else:
|
||||
key += (item["name"].lower(),)
|
||||
return key
|
||||
|
||||
entries.sort(key=get_sort_key, reverse=reverse)
|
||||
total = len(entries)
|
||||
start = (page_num - 1) * page_size
|
||||
end = start + page_size
|
||||
return entries[start:end], total
|
||||
|
||||
async def read_file(self, root: str, rel: str) -> bytes:
|
||||
path = _join_remote(root, rel)
|
||||
|
||||
def _do_read() -> bytes:
|
||||
ftp = self._connect()
|
||||
try:
|
||||
chunks: List[bytes] = []
|
||||
ftp.retrbinary("RETR " + path, lambda b: chunks.append(b))
|
||||
return b"".join(chunks)
|
||||
except error_perm as e:
|
||||
if str(e).startswith("550"):
|
||||
raise FileNotFoundError(rel)
|
||||
raise
|
||||
finally:
|
||||
try:
|
||||
ftp.quit()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return await asyncio.to_thread(_do_read)
|
||||
|
||||
async def write_file(self, root: str, rel: str, data: bytes):
|
||||
path = _join_remote(root, rel)
|
||||
|
||||
def _ensure_dirs(ftp: FTP, dir_path: str):
|
||||
parts = [p for p in dir_path.strip("/").split("/") if p]
|
||||
cur = "/"
|
||||
for p in parts:
|
||||
cur = _join_remote(cur, p)
|
||||
try:
|
||||
ftp.mkd(cur)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _do_write():
|
||||
ftp = self._connect()
|
||||
try:
|
||||
parent = "/" if "/" not in path.strip("/") else path.rsplit("/", 1)[0]
|
||||
_ensure_dirs(ftp, parent)
|
||||
from io import BytesIO
|
||||
bio = BytesIO(data)
|
||||
ftp.storbinary("STOR " + path, bio)
|
||||
finally:
|
||||
try:
|
||||
ftp.quit()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await asyncio.to_thread(_do_write)
|
||||
|
||||
async def write_upload_file(self, root: str, rel: str, file_obj, filename: str | None, file_size: int | None = None, content_type: str | None = None):
|
||||
path = _join_remote(root, rel)
|
||||
|
||||
def _ensure_dirs(ftp: FTP, dir_path: str):
|
||||
parts = [p for p in dir_path.strip("/").split("/") if p]
|
||||
cur = "/"
|
||||
for p in parts:
|
||||
cur = _join_remote(cur, p)
|
||||
try:
|
||||
ftp.mkd(cur)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _do_upload():
|
||||
ftp = self._connect()
|
||||
try:
|
||||
parent = "/" if "/" not in path.strip("/") else path.rsplit("/", 1)[0]
|
||||
_ensure_dirs(ftp, parent)
|
||||
try:
|
||||
if callable(getattr(file_obj, "seek", None)):
|
||||
file_obj.seek(0)
|
||||
except Exception:
|
||||
pass
|
||||
ftp.storbinary("STOR " + path, file_obj)
|
||||
finally:
|
||||
try:
|
||||
ftp.quit()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await asyncio.to_thread(_do_upload)
|
||||
return {"size": file_size or 0}
|
||||
|
||||
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
|
||||
# KISS: 聚合后一次性写入
|
||||
buf = bytearray()
|
||||
async for chunk in data_iter:
|
||||
if chunk:
|
||||
buf.extend(chunk)
|
||||
await self.write_file(root, rel, bytes(buf))
|
||||
return len(buf)
|
||||
|
||||
async def mkdir(self, root: str, rel: str):
|
||||
path = _join_remote(root, rel)
|
||||
|
||||
def _do_mkdir():
|
||||
ftp = self._connect()
|
||||
try:
|
||||
parts = [p for p in path.strip("/").split("/") if p]
|
||||
cur = "/"
|
||||
for p in parts:
|
||||
cur = _join_remote(cur, p)
|
||||
try:
|
||||
ftp.mkd(cur)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
ftp.quit()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await asyncio.to_thread(_do_mkdir)
|
||||
|
||||
async def delete(self, root: str, rel: str):
|
||||
path = _join_remote(root, rel)
|
||||
|
||||
def _do_delete():
|
||||
ftp = self._connect()
|
||||
try:
|
||||
# Try file delete
|
||||
try:
|
||||
ftp.delete(path)
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Recursively delete dir
|
||||
def _rm_tree(dir_path: str):
|
||||
try:
|
||||
ftp.cwd(dir_path)
|
||||
except Exception:
|
||||
return
|
||||
items = []
|
||||
try:
|
||||
for name, facts in ftp.mlsd():
|
||||
if name in (".", ".."):
|
||||
continue
|
||||
items.append((name, facts.get("type") == "dir"))
|
||||
except Exception:
|
||||
try:
|
||||
names = ftp.nlst()
|
||||
except Exception:
|
||||
names = []
|
||||
for n in names:
|
||||
if n in (".", ".."):
|
||||
continue
|
||||
# Best-effort dir check
|
||||
try:
|
||||
ftp.cwd(_join_remote(dir_path, n))
|
||||
ftp.cwd(dir_path)
|
||||
items.append((n, True))
|
||||
except Exception:
|
||||
items.append((n, False))
|
||||
for n, is_dir in items:
|
||||
child = _join_remote(dir_path, n)
|
||||
if is_dir:
|
||||
_rm_tree(child)
|
||||
else:
|
||||
try:
|
||||
ftp.delete(child)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
ftp.rmd(dir_path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
_rm_tree(path)
|
||||
finally:
|
||||
try:
|
||||
ftp.quit()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await asyncio.to_thread(_do_delete)
|
||||
|
||||
async def move(self, root: str, src_rel: str, dst_rel: str):
|
||||
src = _join_remote(root, src_rel)
|
||||
dst = _join_remote(root, dst_rel)
|
||||
|
||||
def _do_move():
|
||||
ftp = self._connect()
|
||||
try:
|
||||
# Ensure dst parent exists
|
||||
parent = "/" if "/" not in dst.strip("/") else dst.rsplit("/", 1)[0]
|
||||
parts = [p for p in parent.strip("/").split("/") if p]
|
||||
cur = "/"
|
||||
for p in parts:
|
||||
cur = _join_remote(cur, p)
|
||||
try:
|
||||
ftp.mkd(cur)
|
||||
except Exception:
|
||||
pass
|
||||
ftp.rename(src, dst)
|
||||
finally:
|
||||
try:
|
||||
ftp.quit()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await asyncio.to_thread(_do_move)
|
||||
|
||||
async def rename(self, root: str, src_rel: str, dst_rel: str):
|
||||
await self.move(root, src_rel, dst_rel)
|
||||
|
||||
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
|
||||
src = _join_remote(root, src_rel)
|
||||
dst = _join_remote(root, dst_rel)
|
||||
|
||||
# naive implementation: download then upload; recursively for dirs
|
||||
async def _is_dir(path: str) -> bool:
|
||||
def _probe() -> bool:
|
||||
ftp = self._connect()
|
||||
try:
|
||||
try:
|
||||
ftp.cwd(path)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
finally:
|
||||
try:
|
||||
ftp.quit()
|
||||
except Exception:
|
||||
pass
|
||||
return await asyncio.to_thread(_probe)
|
||||
|
||||
if await _is_dir(src):
|
||||
# list children, create dst dir, copy recursively
|
||||
await self.mkdir(root, dst_rel)
|
||||
|
||||
children, _ = await self.list_dir(root, src_rel, page_num=1, page_size=10_000)
|
||||
for ent in children:
|
||||
child_src = f"{src_rel.rstrip('/')}/{ent['name']}"
|
||||
child_dst = f"{dst_rel.rstrip('/')}/{ent['name']}"
|
||||
await self.copy(root, child_src, child_dst, overwrite)
|
||||
return
|
||||
|
||||
# file
|
||||
data = await self.read_file(root, src_rel)
|
||||
if not overwrite:
|
||||
# best-effort existence check
|
||||
try:
|
||||
await self.stat_file(root, dst_rel)
|
||||
raise FileExistsError(dst_rel)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
await self.write_file(root, dst_rel, data)
|
||||
|
||||
async def stat_file(self, root: str, rel: str):
|
||||
path = _join_remote(root, rel)
|
||||
|
||||
def _do_stat():
|
||||
ftp = self._connect()
|
||||
try:
|
||||
# Try MLST
|
||||
try:
|
||||
resp: List[str] = []
|
||||
ftp.retrlines("MLST " + path, resp.append)
|
||||
# The last line usually contains facts
|
||||
facts = {}
|
||||
if resp:
|
||||
facts = _parse_mlst_line(resp[-1])
|
||||
name = rel.split("/")[-1]
|
||||
t = facts.get("type") or "file"
|
||||
is_dir = t == "dir"
|
||||
size = int(facts.get("size") or 0)
|
||||
mtime = _parse_modify_to_epoch(facts.get("modify") or "")
|
||||
return {
|
||||
"name": name,
|
||||
"is_dir": is_dir,
|
||||
"size": 0 if is_dir else size,
|
||||
"mtime": mtime,
|
||||
"type": "dir" if is_dir else "file",
|
||||
"path": path,
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Probe directory
|
||||
try:
|
||||
ftp.cwd(path)
|
||||
return {
|
||||
"name": rel.split("/")[-1],
|
||||
"is_dir": True,
|
||||
"size": 0,
|
||||
"mtime": 0,
|
||||
"type": "dir",
|
||||
"path": path,
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Treat as file
|
||||
try:
|
||||
size = ftp.size(path) or 0
|
||||
except Exception:
|
||||
size = 0
|
||||
try:
|
||||
mdtm = ftp.sendcmd("MDTM " + path)
|
||||
mtime = _parse_modify_to_epoch(mdtm.split(" ", 1)[1]) if mdtm.startswith("213 ") else 0
|
||||
except Exception:
|
||||
mtime = 0
|
||||
return {
|
||||
"name": rel.split("/")[-1],
|
||||
"is_dir": False,
|
||||
"size": int(size or 0),
|
||||
"mtime": int(mtime or 0),
|
||||
"type": "file",
|
||||
"path": path,
|
||||
}
|
||||
except error_perm as e:
|
||||
if str(e).startswith("550"):
|
||||
raise FileNotFoundError(rel)
|
||||
raise
|
||||
finally:
|
||||
try:
|
||||
ftp.quit()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return await asyncio.to_thread(_do_stat)
|
||||
|
||||
async def stream_file(self, root: str, rel: str, range_header: str | None):
|
||||
path = _join_remote(root, rel)
|
||||
# Get size (best-effort)
|
||||
def _get_size() -> Optional[int]:
|
||||
ftp = self._connect()
|
||||
try:
|
||||
try:
|
||||
return int(ftp.size(path) or 0)
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
try:
|
||||
ftp.quit()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
total_size = await asyncio.to_thread(_get_size)
|
||||
mime, _ = mimetypes.guess_type(rel)
|
||||
content_type = mime or "application/octet-stream"
|
||||
|
||||
rng: Optional[_Range] = None
|
||||
status = 200
|
||||
headers = {"Accept-Ranges": "bytes", "Content-Type": content_type}
|
||||
if range_header and range_header.startswith("bytes=") and total_size is not None:
|
||||
try:
|
||||
s, e = (range_header.removeprefix("bytes=").split("-", 1))
|
||||
start = int(s) if s.strip() else 0
|
||||
end = int(e) if e.strip() else (total_size - 1)
|
||||
if start >= total_size:
|
||||
raise HTTPException(416, detail="Requested Range Not Satisfiable")
|
||||
if end >= total_size:
|
||||
end = total_size - 1
|
||||
rng = _Range(start, end)
|
||||
status = 206
|
||||
headers["Content-Range"] = f"bytes {start}-{end}/{total_size}"
|
||||
headers["Content-Length"] = str(end - start + 1)
|
||||
except ValueError:
|
||||
raise HTTPException(400, detail="Invalid Range header")
|
||||
elif total_size is not None:
|
||||
headers["Content-Length"] = str(total_size)
|
||||
|
||||
queue: asyncio.Queue[Optional[bytes]] = asyncio.Queue(maxsize=8)
|
||||
|
||||
class _Stop(Exception):
|
||||
pass
|
||||
|
||||
def _worker():
|
||||
ftp = self._connect()
|
||||
remaining = None
|
||||
if rng is not None:
|
||||
remaining = (rng.end - rng.start + 1) if rng.end is not None else None
|
||||
|
||||
def _cb(chunk: bytes):
|
||||
nonlocal remaining
|
||||
if not chunk:
|
||||
return
|
||||
try:
|
||||
if remaining is not None:
|
||||
if len(chunk) > remaining:
|
||||
part = chunk[:remaining]
|
||||
queue.put_nowait(part)
|
||||
remaining = 0
|
||||
raise _Stop()
|
||||
else:
|
||||
queue.put_nowait(chunk)
|
||||
remaining -= len(chunk)
|
||||
if remaining <= 0:
|
||||
raise _Stop()
|
||||
else:
|
||||
queue.put_nowait(chunk)
|
||||
except _Stop:
|
||||
raise
|
||||
except Exception:
|
||||
# queue full or event loop closed
|
||||
raise _Stop()
|
||||
|
||||
try:
|
||||
if rng is not None:
|
||||
ftp.retrbinary("RETR " + path, _cb, rest=rng.start)
|
||||
else:
|
||||
ftp.retrbinary("RETR " + path, _cb)
|
||||
queue.put_nowait(None)
|
||||
except _Stop:
|
||||
try:
|
||||
queue.put_nowait(None)
|
||||
except Exception:
|
||||
pass
|
||||
except error_perm as e:
|
||||
try:
|
||||
queue.put_nowait(None)
|
||||
except Exception:
|
||||
pass
|
||||
if str(e).startswith("550"):
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
ftp.quit()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def agen():
|
||||
worker_fut = asyncio.to_thread(_worker)
|
||||
try:
|
||||
while True:
|
||||
chunk = await queue.get()
|
||||
if chunk is None:
|
||||
break
|
||||
yield chunk
|
||||
finally:
|
||||
try:
|
||||
await worker_fut
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return StreamingResponse(agen(), status_code=status, headers=headers, media_type=content_type)
|
||||
|
||||
|
||||
ADAPTER_TYPE = "ftp"
|
||||
|
||||
CONFIG_SCHEMA = [
|
||||
{"key": "host", "label": "主机", "type": "string", "required": True, "placeholder": "ftp.example.com"},
|
||||
{"key": "port", "label": "端口", "type": "number", "required": False, "default": 21},
|
||||
{"key": "username", "label": "用户名", "type": "string", "required": False},
|
||||
{"key": "password", "label": "密码", "type": "password", "required": False},
|
||||
{"key": "passive", "label": "被动模式", "type": "boolean", "required": False, "default": True},
|
||||
{"key": "timeout", "label": "超时(秒)", "type": "number", "required": False, "default": 15},
|
||||
{"key": "root", "label": "根路径", "type": "string", "required": False, "default": "/"},
|
||||
]
|
||||
|
||||
|
||||
def ADAPTER_FACTORY(rec: StorageAdapter):
|
||||
return FTPAdapter(rec)
|
||||
@@ -0,0 +1,559 @@
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import List, Dict, Tuple, AsyncIterator
|
||||
import httpx
|
||||
from fastapi.responses import StreamingResponse, Response
|
||||
from fastapi import HTTPException
|
||||
from models import StorageAdapter
|
||||
|
||||
GOOGLE_OAUTH_URL = "https://oauth2.googleapis.com/token"
|
||||
GOOGLE_DRIVE_API_URL = "https://www.googleapis.com/drive/v3"
|
||||
|
||||
|
||||
class GoogleDriveAdapter:
|
||||
"""Google Drive 存储适配器"""
|
||||
|
||||
def __init__(self, record: StorageAdapter):
|
||||
self.record = record
|
||||
cfg = record.config
|
||||
self.client_id = cfg.get("client_id")
|
||||
self.client_secret = cfg.get("client_secret")
|
||||
self.refresh_token = cfg.get("refresh_token")
|
||||
self.root_folder_id = cfg.get("root_folder_id", "root")
|
||||
self.enable_redirect_307 = bool(cfg.get("enable_direct_download_307"))
|
||||
|
||||
if not all([self.client_id, self.client_secret, self.refresh_token]):
|
||||
raise ValueError(
|
||||
"Google Drive 适配器需要 client_id, client_secret, 和 refresh_token")
|
||||
|
||||
self._access_token: str | None = None
|
||||
self._token_expiry: datetime | None = None
|
||||
|
||||
def get_effective_root(self, sub_path: str | None) -> str:
|
||||
"""
|
||||
获取有效根路径。
|
||||
:param sub_path: 子路径。
|
||||
:return: 完整的有效路径。
|
||||
"""
|
||||
if sub_path:
|
||||
return f"{sub_path.strip('/')}".strip()
|
||||
return ""
|
||||
|
||||
async def _get_access_token(self) -> str:
|
||||
"""
|
||||
获取或刷新 access token。
|
||||
:return: access token。
|
||||
"""
|
||||
if self._access_token and self._token_expiry and datetime.now(timezone.utc) < self._token_expiry:
|
||||
return self._access_token
|
||||
|
||||
data = {
|
||||
"client_id": self.client_id,
|
||||
"client_secret": self.client_secret,
|
||||
"refresh_token": self.refresh_token,
|
||||
"grant_type": "refresh_token",
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=20.0) as client:
|
||||
resp = await client.post(GOOGLE_OAUTH_URL, data=data)
|
||||
resp.raise_for_status()
|
||||
token_data = resp.json()
|
||||
self._access_token = token_data["access_token"]
|
||||
self._token_expiry = datetime.now(
|
||||
timezone.utc) + timedelta(seconds=token_data["expires_in"] - 300)
|
||||
return self._access_token
|
||||
|
||||
async def _request(self, method: str, endpoint: str, **kwargs):
|
||||
"""
|
||||
向 Google Drive API 发送请求。
|
||||
:param method: HTTP 方法。
|
||||
:param endpoint: API 端点。
|
||||
:param kwargs: 其他请求参数。
|
||||
:return: 响应对象。
|
||||
"""
|
||||
token = await self._get_access_token()
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
if "headers" in kwargs:
|
||||
headers.update(kwargs.pop("headers"))
|
||||
|
||||
url = f"{GOOGLE_DRIVE_API_URL}{endpoint}"
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
resp = await client.request(method, url, headers=headers, **kwargs)
|
||||
if resp.status_code == 401:
|
||||
self._access_token = None
|
||||
token = await self._get_access_token()
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
resp = await client.request(method, url, headers=headers, **kwargs)
|
||||
return resp
|
||||
|
||||
async def _get_folder_id_by_path(self, path: str) -> str:
|
||||
"""
|
||||
通过路径获取文件夹 ID。
|
||||
:param path: 路径。
|
||||
:return: 文件夹 ID。
|
||||
"""
|
||||
if not path or path == "/":
|
||||
return self.root_folder_id
|
||||
|
||||
parts = [p for p in path.strip("/").split("/") if p]
|
||||
current_id = self.root_folder_id
|
||||
|
||||
for part in parts:
|
||||
query = f"name='{part}' and '{current_id}' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false"
|
||||
params = {"q": query, "fields": "files(id, name)"}
|
||||
resp = await self._request("GET", "/files", params=params)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
files = data.get("files", [])
|
||||
if not files:
|
||||
raise FileNotFoundError(f"文件夹不存在: {part}")
|
||||
current_id = files[0]["id"]
|
||||
|
||||
return current_id
|
||||
|
||||
async def _get_file_id_by_path(self, path: str) -> str | None:
|
||||
"""
|
||||
通过路径获取文件 ID。
|
||||
:param path: 文件路径。
|
||||
:return: 文件 ID 或 None。
|
||||
"""
|
||||
if not path or path == "/":
|
||||
return self.root_folder_id
|
||||
|
||||
parts = [p for p in path.strip("/").split("/") if p]
|
||||
parent_id = self.root_folder_id
|
||||
|
||||
for i, part in enumerate(parts):
|
||||
is_last = i == len(parts) - 1
|
||||
mime_filter = "" if is_last else "and mimeType='application/vnd.google-apps.folder'"
|
||||
query = f"name='{part}' and '{parent_id}' in parents {mime_filter} and trashed=false"
|
||||
params = {"q": query, "fields": "files(id, name)"}
|
||||
resp = await self._request("GET", "/files", params=params)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
files = data.get("files", [])
|
||||
if not files:
|
||||
return None
|
||||
parent_id = files[0]["id"]
|
||||
|
||||
return parent_id
|
||||
|
||||
def _format_item(self, item: Dict) -> Dict:
|
||||
"""
|
||||
将 Google Drive API 返回的 item 格式化为统一的格式。
|
||||
:param item: Google Drive API 返回的 item 字典。
|
||||
:return: 格式化后的字典。
|
||||
"""
|
||||
is_dir = item["mimeType"] == "application/vnd.google-apps.folder"
|
||||
mtime_str = item.get("modifiedTime", item.get("createdTime", ""))
|
||||
try:
|
||||
mtime = int(datetime.fromisoformat(mtime_str.replace("Z", "+00:00")).timestamp())
|
||||
except:
|
||||
mtime = 0
|
||||
|
||||
return {
|
||||
"name": item["name"],
|
||||
"is_dir": is_dir,
|
||||
"size": 0 if is_dir else int(item.get("size", 0)),
|
||||
"mtime": mtime,
|
||||
"type": "dir" if is_dir else "file",
|
||||
}
|
||||
|
||||
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
|
||||
"""
|
||||
列出目录内容。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
:param page_num: 页码。
|
||||
:param page_size: 每页大小。
|
||||
:param sort_by: 排序字段
|
||||
:param sort_order: 排序顺序
|
||||
:return: 文件/目录列表和总数。
|
||||
"""
|
||||
try:
|
||||
folder_id = await self._get_folder_id_by_path(rel)
|
||||
except FileNotFoundError:
|
||||
return [], 0
|
||||
|
||||
query = f"'{folder_id}' in parents and trashed=false"
|
||||
params = {
|
||||
"q": query,
|
||||
"fields": "files(id, name, mimeType, size, modifiedTime, createdTime)",
|
||||
"pageSize": 1000,
|
||||
}
|
||||
|
||||
all_items = []
|
||||
page_token = None
|
||||
|
||||
while True:
|
||||
if page_token:
|
||||
params["pageToken"] = page_token
|
||||
|
||||
resp = await self._request("GET", "/files", params=params)
|
||||
if resp.status_code == 404:
|
||||
return [], 0
|
||||
resp.raise_for_status()
|
||||
|
||||
data = resp.json()
|
||||
all_items.extend(data.get("files", []))
|
||||
page_token = data.get("nextPageToken")
|
||||
|
||||
if not page_token:
|
||||
break
|
||||
|
||||
formatted_items = [self._format_item(item) for item in all_items]
|
||||
|
||||
# 排序
|
||||
reverse = sort_order.lower() == "desc"
|
||||
def get_sort_key(item):
|
||||
key = (not item["is_dir"],)
|
||||
sort_field = sort_by.lower()
|
||||
if sort_field == "name":
|
||||
key += (item["name"].lower(),)
|
||||
elif sort_field == "size":
|
||||
key += (item["size"],)
|
||||
elif sort_field == "mtime":
|
||||
key += (item["mtime"],)
|
||||
else:
|
||||
key += (item["name"].lower(),)
|
||||
return key
|
||||
formatted_items.sort(key=get_sort_key, reverse=reverse)
|
||||
|
||||
total_count = len(formatted_items)
|
||||
start_idx = (page_num - 1) * page_size
|
||||
end_idx = start_idx + page_size
|
||||
|
||||
return formatted_items[start_idx:end_idx], total_count
|
||||
|
||||
async def read_file(self, root: str, rel: str) -> bytes:
|
||||
"""
|
||||
读取文件内容。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
:return: 文件内容的字节流。
|
||||
"""
|
||||
file_id = await self._get_file_id_by_path(rel)
|
||||
if not file_id:
|
||||
raise FileNotFoundError(rel)
|
||||
|
||||
resp = await self._request("GET", f"/files/{file_id}", params={"alt": "media"})
|
||||
if resp.status_code == 404:
|
||||
raise FileNotFoundError(rel)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
|
||||
async def write_file(self, root: str, rel: str, data: bytes):
|
||||
"""
|
||||
写入文件。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
:param data: 文件内容的字节流。
|
||||
"""
|
||||
parent_path = "/".join(rel.strip("/").split("/")[:-1])
|
||||
file_name = rel.strip("/").split("/")[-1]
|
||||
parent_id = await self._get_folder_id_by_path(parent_path)
|
||||
|
||||
# 检查文件是否已存在
|
||||
existing_id = await self._get_file_id_by_path(rel)
|
||||
|
||||
if existing_id:
|
||||
# 更新现有文件
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
token = await self._get_access_token()
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
url = f"https://www.googleapis.com/upload/drive/v3/files/{existing_id}?uploadType=media"
|
||||
resp = await client.patch(url, headers=headers, content=data)
|
||||
resp.raise_for_status()
|
||||
else:
|
||||
# 创建新文件
|
||||
metadata = {
|
||||
"name": file_name,
|
||||
"parents": [parent_id]
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
token = await self._get_access_token()
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
|
||||
# 使用 multipart 上传
|
||||
import json
|
||||
boundary = "===============boundary==============="
|
||||
headers["Content-Type"] = f"multipart/related; boundary={boundary}"
|
||||
|
||||
body = (
|
||||
f"--{boundary}\r\n"
|
||||
f"Content-Type: application/json; charset=UTF-8\r\n\r\n"
|
||||
f"{json.dumps(metadata)}\r\n"
|
||||
f"--{boundary}\r\n"
|
||||
f"Content-Type: application/octet-stream\r\n\r\n"
|
||||
).encode() + data + f"\r\n--{boundary}--".encode()
|
||||
|
||||
url = "https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart"
|
||||
resp = await client.post(url, headers=headers, content=body)
|
||||
resp.raise_for_status()
|
||||
|
||||
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
|
||||
"""
|
||||
以流式方式写入文件。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
:param data_iter: 文件内容的异步迭代器。
|
||||
:return: 文件大小。
|
||||
"""
|
||||
# 先收集所有数据
|
||||
chunks = []
|
||||
total_size = 0
|
||||
async for chunk in data_iter:
|
||||
chunks.append(chunk)
|
||||
total_size += len(chunk)
|
||||
|
||||
data = b"".join(chunks)
|
||||
await self.write_file(root, rel, data)
|
||||
return total_size
|
||||
|
||||
async def mkdir(self, root: str, rel: str):
|
||||
"""
|
||||
创建目录。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
"""
|
||||
parent_path = "/".join(rel.strip("/").split("/")[:-1])
|
||||
folder_name = rel.strip("/").split("/")[-1]
|
||||
parent_id = await self._get_folder_id_by_path(parent_path)
|
||||
|
||||
metadata = {
|
||||
"name": folder_name,
|
||||
"mimeType": "application/vnd.google-apps.folder",
|
||||
"parents": [parent_id]
|
||||
}
|
||||
|
||||
resp = await self._request("POST", "/files", json=metadata)
|
||||
resp.raise_for_status()
|
||||
|
||||
async def delete(self, root: str, rel: str):
|
||||
"""
|
||||
删除文件或目录。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
"""
|
||||
file_id = await self._get_file_id_by_path(rel)
|
||||
if not file_id:
|
||||
return
|
||||
|
||||
resp = await self._request("DELETE", f"/files/{file_id}")
|
||||
if resp.status_code not in (204, 404):
|
||||
resp.raise_for_status()
|
||||
|
||||
async def move(self, root: str, src_rel: str, dst_rel: str):
|
||||
"""
|
||||
移动或重命名文件/目录。
|
||||
:param root: 根路径。
|
||||
:param src_rel: 源相对路径。
|
||||
:param dst_rel: 目标相对路径。
|
||||
"""
|
||||
file_id = await self._get_file_id_by_path(src_rel)
|
||||
if not file_id:
|
||||
raise FileNotFoundError(src_rel)
|
||||
|
||||
# 获取当前父文件夹
|
||||
resp = await self._request("GET", f"/files/{file_id}", params={"fields": "parents"})
|
||||
resp.raise_for_status()
|
||||
current_parents = resp.json().get("parents", [])
|
||||
|
||||
# 获取目标父文件夹和新名称
|
||||
dst_parent_path = "/".join(dst_rel.strip("/").split("/")[:-1])
|
||||
dst_name = dst_rel.strip("/").split("/")[-1]
|
||||
dst_parent_id = await self._get_folder_id_by_path(dst_parent_path)
|
||||
|
||||
# 更新文件
|
||||
params = {
|
||||
"addParents": dst_parent_id,
|
||||
"removeParents": ",".join(current_parents) if current_parents else None,
|
||||
}
|
||||
metadata = {"name": dst_name}
|
||||
|
||||
resp = await self._request("PATCH", f"/files/{file_id}", params=params, json=metadata)
|
||||
resp.raise_for_status()
|
||||
|
||||
async def rename(self, root: str, src_rel: str, dst_rel: str):
|
||||
"""
|
||||
重命名文件或目录。
|
||||
"""
|
||||
await self.move(root, src_rel, dst_rel)
|
||||
|
||||
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
|
||||
"""
|
||||
复制文件或目录。
|
||||
:param root: 根路径。
|
||||
:param src_rel: 源相对路径。
|
||||
:param dst_rel: 目标相对路径。
|
||||
:param overwrite: 是否覆盖。
|
||||
"""
|
||||
file_id = await self._get_file_id_by_path(src_rel)
|
||||
if not file_id:
|
||||
raise FileNotFoundError(src_rel)
|
||||
|
||||
dst_parent_path = "/".join(dst_rel.strip("/").split("/")[:-1])
|
||||
dst_name = dst_rel.strip("/").split("/")[-1]
|
||||
dst_parent_id = await self._get_folder_id_by_path(dst_parent_path)
|
||||
|
||||
metadata = {
|
||||
"name": dst_name,
|
||||
"parents": [dst_parent_id]
|
||||
}
|
||||
|
||||
resp = await self._request("POST", f"/files/{file_id}/copy", json=metadata)
|
||||
resp.raise_for_status()
|
||||
|
||||
async def stream_file(self, root: str, rel: str, range_header: str | None):
|
||||
"""
|
||||
流式传输文件(支持范围请求)。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
:param range_header: HTTP Range 头。
|
||||
:return: FastAPI StreamingResponse 对象。
|
||||
"""
|
||||
file_id = await self._get_file_id_by_path(rel)
|
||||
if not file_id:
|
||||
raise FileNotFoundError(rel)
|
||||
|
||||
# 获取文件元数据
|
||||
resp = await self._request("GET", f"/files/{file_id}", params={"fields": "name, size, mimeType"})
|
||||
if resp.status_code == 404:
|
||||
raise FileNotFoundError(rel)
|
||||
resp.raise_for_status()
|
||||
item_data = resp.json()
|
||||
|
||||
file_size = int(item_data.get("size", 0))
|
||||
content_type = item_data.get("mimeType", "application/octet-stream")
|
||||
|
||||
start = 0
|
||||
end = file_size - 1
|
||||
status = 200
|
||||
headers = {
|
||||
"Accept-Ranges": "bytes",
|
||||
"Content-Type": content_type,
|
||||
"Content-Disposition": f"inline; filename=\"{item_data.get('name')}\""
|
||||
}
|
||||
|
||||
if range_header and range_header.startswith("bytes="):
|
||||
try:
|
||||
part = range_header.removeprefix("bytes=")
|
||||
s, e = part.split("-", 1)
|
||||
if s.strip():
|
||||
start = int(s)
|
||||
if e.strip():
|
||||
end = int(e)
|
||||
if start >= file_size:
|
||||
raise HTTPException(416, "Requested Range Not Satisfiable")
|
||||
if end >= file_size:
|
||||
end = file_size - 1
|
||||
status = 206
|
||||
except ValueError:
|
||||
raise HTTPException(400, "Invalid Range header")
|
||||
|
||||
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
|
||||
headers["Content-Length"] = str(end - start + 1)
|
||||
else:
|
||||
headers["Content-Length"] = str(file_size)
|
||||
|
||||
async def file_iterator():
|
||||
nonlocal start, end
|
||||
token = await self._get_access_token()
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
req_headers = {
|
||||
'Authorization': f'Bearer {token}',
|
||||
'Range': f'bytes={start}-{end}'
|
||||
}
|
||||
url = f"{GOOGLE_DRIVE_API_URL}/files/{file_id}?alt=media"
|
||||
async with client.stream("GET", url, headers=req_headers) as stream_resp:
|
||||
stream_resp.raise_for_status()
|
||||
async for chunk in stream_resp.aiter_bytes():
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(file_iterator(), status_code=status, headers=headers, media_type=content_type)
|
||||
|
||||
async def stat_file(self, root: str, rel: str):
|
||||
"""
|
||||
获取文件或目录的元数据。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
:return: 格式化后的文件/目录信息。
|
||||
"""
|
||||
file_id = await self._get_file_id_by_path(rel)
|
||||
if not file_id:
|
||||
raise FileNotFoundError(rel)
|
||||
|
||||
resp = await self._request("GET", f"/files/{file_id}", params={"fields": "id, name, mimeType, size, modifiedTime, createdTime"})
|
||||
if resp.status_code == 404:
|
||||
raise FileNotFoundError(rel)
|
||||
resp.raise_for_status()
|
||||
return self._format_item(resp.json())
|
||||
|
||||
async def get_direct_download_response(self, root: str, rel: str):
|
||||
"""
|
||||
获取直接下载响应 (307 重定向)。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
:return: 307 重定向响应或 None。
|
||||
"""
|
||||
if not self.enable_redirect_307:
|
||||
return None
|
||||
|
||||
file_id = await self._get_file_id_by_path(rel)
|
||||
if not file_id:
|
||||
raise FileNotFoundError(rel)
|
||||
|
||||
# 获取文件的下载链接
|
||||
resp = await self._request("GET", f"/files/{file_id}", params={"fields": "webContentLink"})
|
||||
if resp.status_code == 404:
|
||||
raise FileNotFoundError(rel)
|
||||
resp.raise_for_status()
|
||||
|
||||
item_data = resp.json()
|
||||
download_url = item_data.get("webContentLink")
|
||||
if not download_url:
|
||||
return None
|
||||
|
||||
return Response(status_code=307, headers={"Location": download_url})
|
||||
|
||||
async def get_thumbnail(self, root: str, rel: str, size: str = "medium"):
|
||||
"""
|
||||
获取文件的缩略图。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
:param size: 缩略图大小 (暂未使用,Google Drive 自动决定)。
|
||||
:return: 缩略图内容的字节流,或在不支持时返回 None。
|
||||
"""
|
||||
file_id = await self._get_file_id_by_path(rel)
|
||||
if not file_id:
|
||||
return None
|
||||
|
||||
try:
|
||||
resp = await self._request("GET", f"/files/{file_id}", params={"fields": "thumbnailLink"})
|
||||
if resp.status_code == 200:
|
||||
item_data = resp.json()
|
||||
thumbnail_link = item_data.get("thumbnailLink")
|
||||
if thumbnail_link:
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
thumb_resp = await client.get(thumbnail_link)
|
||||
thumb_resp.raise_for_status()
|
||||
return thumb_resp.content
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
ADAPTER_TYPE = "googledrive"
|
||||
|
||||
CONFIG_SCHEMA = [
|
||||
{"key": "client_id", "label": "Client ID", "type": "string", "required": True},
|
||||
{"key": "client_secret", "label": "Client Secret",
|
||||
"type": "password", "required": True},
|
||||
{"key": "refresh_token", "label": "Refresh Token", "type": "password",
|
||||
"required": True, "help_text": "可以通过 Google OAuth 2.0 Playground 获取"},
|
||||
{"key": "root_folder_id", "label": "根文件夹 ID (Root Folder ID)", "type": "string",
|
||||
"required": False, "placeholder": "默认为根目录 (root)", "default": "root"},
|
||||
{"key": "enable_direct_download_307", "label": "Enable 307 redirect download", "type": "boolean", "default": False},
|
||||
]
|
||||
|
||||
|
||||
def ADAPTER_FACTORY(rec): return GoogleDriveAdapter(rec)
|
||||
@@ -0,0 +1,337 @@
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Tuple, AsyncIterator
|
||||
import asyncio
|
||||
import mimetypes
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import StreamingResponse, Response
|
||||
from models import StorageAdapter
|
||||
|
||||
|
||||
def _safe_join(root: str, rel: str) -> Path:
|
||||
root_path = Path(root).resolve()
|
||||
full = (root_path / rel).resolve()
|
||||
if not str(full).startswith(str(root_path)):
|
||||
raise ValueError("Path escape detected")
|
||||
return full
|
||||
|
||||
|
||||
DEFAULT_FILE_MODE = 0o666
|
||||
DEFAULT_DIR_MODE = 0o777
|
||||
|
||||
|
||||
def _apply_mode(path: Path, mode: int):
|
||||
try:
|
||||
os.chmod(path, mode)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class LocalAdapter:
|
||||
def __init__(self, record: StorageAdapter):
|
||||
self.record = record
|
||||
self.root = self.record.config.get("root")
|
||||
if not self.root:
|
||||
raise ValueError("Local adapter config requires 'root'")
|
||||
Path(self.root).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def get_effective_root(self, sub_path: str | None) -> str:
|
||||
root = self.record.config.get("root")
|
||||
if sub_path:
|
||||
return str(Path(root) / sub_path)
|
||||
return root
|
||||
|
||||
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
|
||||
rel = rel.strip('/')
|
||||
base = _safe_join(root, rel) if rel else Path(root)
|
||||
if not base.exists():
|
||||
return [], 0
|
||||
if not base.is_dir():
|
||||
raise NotADirectoryError(rel)
|
||||
|
||||
all_names = await asyncio.to_thread(os.listdir, base)
|
||||
|
||||
entries = []
|
||||
for name in all_names:
|
||||
fp = base / name
|
||||
try:
|
||||
st = await asyncio.to_thread(fp.stat)
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
is_dir = fp.is_dir()
|
||||
entries.append({
|
||||
"name": name,
|
||||
"is_dir": is_dir,
|
||||
"size": 0 if is_dir else st.st_size,
|
||||
"mtime": int(st.st_mtime),
|
||||
"mode": stat.S_IMODE(st.st_mode),
|
||||
"type": "dir" if is_dir else "file",
|
||||
})
|
||||
|
||||
# 排序
|
||||
reverse = sort_order.lower() == "desc"
|
||||
|
||||
def get_sort_key(item):
|
||||
# 基础排序键,目录优先
|
||||
key = (not item["is_dir"],)
|
||||
sort_field = sort_by.lower()
|
||||
|
||||
if sort_field == "name":
|
||||
key += (item["name"].lower(),)
|
||||
elif sort_field == "size":
|
||||
key += (item["size"],)
|
||||
elif sort_field == "mtime":
|
||||
key += (item["mtime"],)
|
||||
else: # 默认按名称
|
||||
key += (item["name"].lower(),)
|
||||
return key
|
||||
|
||||
entries.sort(key=get_sort_key, reverse=reverse)
|
||||
|
||||
total_count = len(entries)
|
||||
|
||||
# 分页
|
||||
start_idx = (page_num - 1) * page_size
|
||||
end_idx = start_idx + page_size
|
||||
page_entries = entries[start_idx:end_idx]
|
||||
|
||||
return page_entries, total_count
|
||||
|
||||
async def read_file(self, root: str, rel: str) -> bytes:
|
||||
fp = _safe_join(root, rel)
|
||||
if not fp.exists() or not fp.is_file():
|
||||
raise FileNotFoundError(rel)
|
||||
return await asyncio.to_thread(fp.read_bytes)
|
||||
|
||||
async def write_file(self, root: str, rel: str, data: bytes):
|
||||
fp = _safe_join(root, rel)
|
||||
pre_exists = fp.exists()
|
||||
await asyncio.to_thread(os.makedirs, fp.parent, mode=DEFAULT_DIR_MODE, exist_ok=True)
|
||||
await asyncio.to_thread(fp.write_bytes, data)
|
||||
if not pre_exists:
|
||||
await asyncio.to_thread(_apply_mode, fp, DEFAULT_FILE_MODE)
|
||||
|
||||
async def write_upload_file(self, root: str, rel: str, file_obj, filename: str | None, file_size: int | None = None, content_type: str | None = None):
|
||||
fp = _safe_join(root, rel)
|
||||
pre_exists = fp.exists()
|
||||
await asyncio.to_thread(os.makedirs, fp.parent, mode=DEFAULT_DIR_MODE, exist_ok=True)
|
||||
|
||||
def _copy():
|
||||
try:
|
||||
if callable(getattr(file_obj, "seek", None)):
|
||||
file_obj.seek(0)
|
||||
except Exception:
|
||||
pass
|
||||
with open(fp, "wb") as f:
|
||||
shutil.copyfileobj(file_obj, f)
|
||||
|
||||
await asyncio.to_thread(_copy)
|
||||
if not pre_exists:
|
||||
await asyncio.to_thread(_apply_mode, fp, DEFAULT_FILE_MODE)
|
||||
|
||||
size = file_size
|
||||
if size is None:
|
||||
try:
|
||||
size = fp.stat().st_size
|
||||
except Exception:
|
||||
size = 0
|
||||
return {"size": int(size or 0)}
|
||||
|
||||
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
|
||||
fp = _safe_join(root, rel)
|
||||
pre_exists = fp.exists()
|
||||
await asyncio.to_thread(os.makedirs, fp.parent, mode=DEFAULT_DIR_MODE, exist_ok=True)
|
||||
# 流式写入,避免一次性读入内存
|
||||
def _open():
|
||||
return open(fp, "wb")
|
||||
f = await asyncio.to_thread(_open)
|
||||
size = 0
|
||||
try:
|
||||
async for chunk in data_iter:
|
||||
if not chunk:
|
||||
continue
|
||||
size += len(chunk)
|
||||
await asyncio.to_thread(f.write, chunk)
|
||||
finally:
|
||||
await asyncio.to_thread(f.close)
|
||||
if not pre_exists:
|
||||
await asyncio.to_thread(_apply_mode, fp, DEFAULT_FILE_MODE)
|
||||
return size
|
||||
|
||||
async def mkdir(self, root: str, rel: str):
|
||||
fp = _safe_join(root, rel)
|
||||
await asyncio.to_thread(os.makedirs, fp, mode=DEFAULT_DIR_MODE, exist_ok=True)
|
||||
|
||||
async def delete(self, root: str, rel: str):
|
||||
fp = _safe_join(root, rel)
|
||||
if not fp.exists():
|
||||
return
|
||||
if fp.is_dir():
|
||||
await asyncio.to_thread(shutil.rmtree, fp)
|
||||
else:
|
||||
await asyncio.to_thread(fp.unlink)
|
||||
|
||||
async def stat_path(self, root: str, rel: str):
|
||||
"""新增: 返回路径状态调试信息"""
|
||||
fp = _safe_join(root, rel)
|
||||
def _stat():
|
||||
if not fp.exists():
|
||||
return {"exists": False, "is_dir": None, "path": str(fp)}
|
||||
return {
|
||||
"exists": True,
|
||||
"is_dir": fp.is_dir(),
|
||||
"path": str(fp)
|
||||
}
|
||||
return await asyncio.to_thread(_stat)
|
||||
|
||||
async def exists(self, root: str, rel: str) -> bool:
|
||||
"""新增: 判断路径是否存在"""
|
||||
fp = _safe_join(root, rel)
|
||||
return await asyncio.to_thread(fp.exists)
|
||||
|
||||
async def move(self, root: str, src_rel: str, dst_rel: str):
|
||||
src = _safe_join(root, src_rel)
|
||||
dst = _safe_join(root, dst_rel)
|
||||
if str(src) == str(dst):
|
||||
return
|
||||
if not src.exists():
|
||||
raise FileNotFoundError(src_rel)
|
||||
await asyncio.to_thread(dst.parent.mkdir, parents=True, exist_ok=True)
|
||||
|
||||
def _do_move():
|
||||
try:
|
||||
os.replace(src, dst)
|
||||
except OSError:
|
||||
shutil.move(str(src), str(dst))
|
||||
await asyncio.to_thread(_do_move)
|
||||
|
||||
async def rename(self, root: str, src_rel: str, dst_rel: str):
|
||||
src = _safe_join(root, src_rel)
|
||||
dst = _safe_join(root, dst_rel)
|
||||
if str(src) == str(dst):
|
||||
return
|
||||
if not src.exists():
|
||||
raise FileNotFoundError(src_rel)
|
||||
await asyncio.to_thread(dst.parent.mkdir, parents=True, exist_ok=True)
|
||||
def _do_rename():
|
||||
try:
|
||||
os.rename(src, dst)
|
||||
except OSError:
|
||||
os.replace(src, dst)
|
||||
await asyncio.to_thread(_do_rename)
|
||||
|
||||
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
|
||||
src = _safe_join(root, src_rel)
|
||||
dst = _safe_join(root, dst_rel)
|
||||
if not src.exists():
|
||||
raise FileNotFoundError(src_rel)
|
||||
if str(src) == str(dst):
|
||||
return
|
||||
await asyncio.to_thread(dst.parent.mkdir, parents=True, exist_ok=True)
|
||||
def _do():
|
||||
if dst.exists():
|
||||
if not overwrite:
|
||||
raise FileExistsError(dst_rel)
|
||||
if dst.is_dir():
|
||||
shutil.rmtree(dst)
|
||||
else:
|
||||
dst.unlink()
|
||||
if src.is_dir():
|
||||
shutil.copytree(src, dst)
|
||||
else:
|
||||
shutil.copy2(src, dst)
|
||||
await asyncio.to_thread(_do)
|
||||
|
||||
async def stream_file(self, root: str, rel: str, range_header: str | None):
|
||||
fp = _safe_join(root, rel)
|
||||
if not fp.exists() or not fp.is_file():
|
||||
raise HTTPException(404, detail="File not found")
|
||||
mime, _ = mimetypes.guess_type(rel)
|
||||
content_type = mime or "application/octet-stream"
|
||||
file_size = (await asyncio.to_thread(fp.stat)).st_size
|
||||
start = 0
|
||||
end = file_size - 1
|
||||
status = 200
|
||||
headers = {
|
||||
"Accept-Ranges": "bytes",
|
||||
"Content-Type": content_type,
|
||||
}
|
||||
if range_header and range_header.startswith("bytes="):
|
||||
try:
|
||||
part = range_header.removeprefix("bytes=")
|
||||
s, e = part.split("-", 1)
|
||||
if s.strip():
|
||||
start = int(s)
|
||||
if e.strip():
|
||||
end = int(e)
|
||||
if start >= file_size:
|
||||
raise HTTPException(416, detail="Requested Range Not Satisfiable")
|
||||
if end >= file_size:
|
||||
end = file_size - 1
|
||||
status = 206
|
||||
except ValueError:
|
||||
raise HTTPException(400, detail="Invalid Range header")
|
||||
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
|
||||
headers["Content-Length"] = str(end - start + 1)
|
||||
else:
|
||||
headers["Content-Length"] = str(file_size)
|
||||
|
||||
async def iterator():
|
||||
# 使用线程池避免阻塞
|
||||
def _read_segment(offset: int, length: int):
|
||||
with open(fp, "rb") as f:
|
||||
f.seek(offset)
|
||||
return f.read(length)
|
||||
chunk_size = 256 * 1024
|
||||
remaining = end - start + 1
|
||||
offset = start
|
||||
while remaining > 0:
|
||||
size = min(chunk_size, remaining)
|
||||
data = await asyncio.to_thread(_read_segment, offset, size)
|
||||
if not data:
|
||||
break
|
||||
yield data
|
||||
remaining -= len(data)
|
||||
offset += len(data)
|
||||
|
||||
return StreamingResponse(iterator(), status_code=status, headers=headers, media_type=content_type)
|
||||
|
||||
async def stat_file(self, root: str, rel: str):
|
||||
fp = _safe_join(root, rel)
|
||||
if not fp.exists():
|
||||
raise FileNotFoundError(rel)
|
||||
st = await asyncio.to_thread(fp.stat)
|
||||
info = {
|
||||
"name": fp.name,
|
||||
"is_dir": fp.is_dir(),
|
||||
"size": st.st_size,
|
||||
"mtime": int(st.st_mtime),
|
||||
"mode": stat.S_IMODE(st.st_mode),
|
||||
"type": "dir" if fp.is_dir() else "file",
|
||||
"path": str(fp),
|
||||
}
|
||||
# exif信息
|
||||
exif = None
|
||||
if not fp.is_dir():
|
||||
mime, _ = mimetypes.guess_type(fp.name)
|
||||
if mime and mime.startswith("image/"):
|
||||
try:
|
||||
from PIL import Image
|
||||
img = await asyncio.to_thread(Image.open, fp)
|
||||
exif_data = img._getexif()
|
||||
if exif_data:
|
||||
exif = {str(k): str(v) for k, v in exif_data.items()}
|
||||
except Exception:
|
||||
exif = None
|
||||
info["exif"] = exif
|
||||
return info
|
||||
|
||||
|
||||
ADAPTER_TYPE = "local"
|
||||
CONFIG_SCHEMA = [
|
||||
{"key": "root", "label": "根目录", "type": "string", "required": True, "placeholder": "/data/storage"},
|
||||
]
|
||||
ADAPTER_FACTORY = lambda rec: LocalAdapter(rec)
|
||||
@@ -0,0 +1,461 @@
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import List, Dict, Tuple, AsyncIterator
|
||||
import httpx
|
||||
from fastapi.responses import StreamingResponse, Response
|
||||
from fastapi import HTTPException
|
||||
from models import StorageAdapter
|
||||
|
||||
MS_GRAPH_URL = "https://graph.microsoft.com/v1.0"
|
||||
MS_OAUTH_URL = "https://login.microsoftonline.com/common/oauth2/v2.0/token"
|
||||
|
||||
|
||||
class OneDriveAdapter:
|
||||
"""OneDrive 存储适配器"""
|
||||
|
||||
def __init__(self, record: StorageAdapter):
|
||||
self.record = record
|
||||
cfg = record.config
|
||||
self.client_id = cfg.get("client_id")
|
||||
self.client_secret = cfg.get("client_secret")
|
||||
self.refresh_token = cfg.get("refresh_token")
|
||||
self.root = cfg.get("root", "/").strip("/")
|
||||
self.enable_redirect_307 = bool(cfg.get("enable_direct_download_307"))
|
||||
|
||||
if not all([self.client_id, self.client_secret, self.refresh_token]):
|
||||
raise ValueError(
|
||||
"OneDrive 适配器需要 client_id, client_secret, 和 refresh_token")
|
||||
|
||||
self._access_token: str | None = None
|
||||
self._token_expiry: datetime | None = None
|
||||
|
||||
def get_effective_root(self, sub_path: str | None) -> str:
|
||||
"""
|
||||
获取有效根路径。
|
||||
:param sub_path: 子路径。
|
||||
:return: 完整的有效路径。
|
||||
"""
|
||||
if sub_path:
|
||||
return f"/{self.root.strip('/')}/{sub_path.strip('/')}".strip()
|
||||
return f"/{self.root.strip('/')}".strip()
|
||||
|
||||
def _get_api_path(self, rel_path: str) -> str:
|
||||
"""
|
||||
将用户可见的相对路径转换为 Graph API 路径段。
|
||||
:param rel_path: 相对路径。
|
||||
:return: Graph API 路径段。
|
||||
"""
|
||||
full_path = self.get_effective_root(rel_path).strip('/')
|
||||
if not full_path:
|
||||
return ""
|
||||
return f":/{full_path}"
|
||||
|
||||
async def _get_access_token(self) -> str:
|
||||
"""
|
||||
获取或刷新 access token。
|
||||
:return: access token。
|
||||
"""
|
||||
if self._access_token and self._token_expiry and datetime.now(timezone.utc) < self._token_expiry:
|
||||
return self._access_token
|
||||
|
||||
data = {
|
||||
"client_id": self.client_id,
|
||||
"client_secret": self.client_secret,
|
||||
"refresh_token": self.refresh_token,
|
||||
"grant_type": "refresh_token",
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=20.0) as client:
|
||||
resp = await client.post(MS_OAUTH_URL, data=data)
|
||||
resp.raise_for_status()
|
||||
token_data = resp.json()
|
||||
self._access_token = token_data["access_token"]
|
||||
self._token_expiry = datetime.now(
|
||||
timezone.utc) + timedelta(seconds=token_data["expires_in"] - 300)
|
||||
return self._access_token
|
||||
|
||||
async def _request(self, method: str, api_path_segment: str | None = None, *, full_url: str | None = None, **kwargs):
|
||||
"""
|
||||
向 Microsoft Graph API 发送请求。
|
||||
:param method: HTTP 方法。
|
||||
:param api_path_segment: API 路径段 (与 full_url 互斥)。
|
||||
:param full_url: 完整的请求 URL (与 api_path_segment 互斥)。
|
||||
:param kwargs: 其他请求参数。
|
||||
:return: 响应对象。
|
||||
"""
|
||||
if not ((api_path_segment is not None) ^ (full_url is not None)):
|
||||
raise ValueError("必须提供 api_path_segment 或 full_url 中的一个,且仅一个")
|
||||
|
||||
token = await self._get_access_token()
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
if "headers" in kwargs:
|
||||
headers.update(kwargs.pop("headers"))
|
||||
|
||||
url = full_url if full_url else f"{MS_GRAPH_URL}/me/drive/root{api_path_segment}"
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
resp = await client.request(method, url, headers=headers, **kwargs)
|
||||
if resp.status_code == 401:
|
||||
self._access_token = None
|
||||
token = await self._get_access_token()
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
resp = await client.request(method, url, headers=headers, **kwargs)
|
||||
return resp
|
||||
|
||||
def _format_item(self, item: Dict) -> Dict:
|
||||
"""
|
||||
将 Graph API 返回的 item 格式化为统一的格式。
|
||||
:param item: Graph API 返回的 item 字典。
|
||||
:return: 格式化后的字典。
|
||||
"""
|
||||
is_dir = "folder" in item
|
||||
return {
|
||||
"name": item["name"],
|
||||
"is_dir": is_dir,
|
||||
"size": 0 if is_dir else item.get("size", 0),
|
||||
"mtime": int(datetime.fromisoformat(item["lastModifiedDateTime"].replace("Z", "+00:00")).timestamp()),
|
||||
"type": "dir" if is_dir else "file",
|
||||
}
|
||||
|
||||
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
|
||||
"""
|
||||
列出目录内容。
|
||||
由于 Graph API 不支持基于偏移($skip)的分页,此方法将获取所有项目,
|
||||
:param root: 根路径 (在此适配器中未使用,通过配置的 root 确定)。
|
||||
:param rel: 相对路径。
|
||||
:param page_num: 页码。
|
||||
:param page_size: 每页大小。
|
||||
:param sort_by: 排序字段
|
||||
:param sort_order: 排序顺序
|
||||
:return: 文件/目录列表和总数。
|
||||
"""
|
||||
api_path = self._get_api_path(rel)
|
||||
children_path = f"{api_path}:/children" if api_path else "/children"
|
||||
all_items = []
|
||||
params = {"$top": 999}
|
||||
resp = await self._request("GET", api_path_segment=children_path, params=params)
|
||||
|
||||
while True:
|
||||
if resp.status_code == 404 and not all_items:
|
||||
return [], 0
|
||||
resp.raise_for_status()
|
||||
|
||||
try:
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
raise IOError(f"解析 Graph API 响应失败: {e}") from e
|
||||
|
||||
all_items.extend(data.get("value", []))
|
||||
next_link = data.get("@odata.nextLink")
|
||||
|
||||
if not next_link:
|
||||
break
|
||||
|
||||
resp = await self._request("GET", full_url=next_link)
|
||||
|
||||
formatted_items = [self._format_item(item) for item in all_items]
|
||||
|
||||
# 排序
|
||||
reverse = sort_order.lower() == "desc"
|
||||
def get_sort_key(item):
|
||||
key = (not item["is_dir"],)
|
||||
sort_field = sort_by.lower()
|
||||
if sort_field == "name":
|
||||
key += (item["name"].lower(),)
|
||||
elif sort_field == "size":
|
||||
key += (item["size"],)
|
||||
elif sort_field == "mtime":
|
||||
key += (item["mtime"],)
|
||||
else:
|
||||
key += (item["name"].lower(),)
|
||||
return key
|
||||
formatted_items.sort(key=get_sort_key, reverse=reverse)
|
||||
|
||||
total_count = len(formatted_items)
|
||||
start_idx = (page_num - 1) * page_size
|
||||
end_idx = start_idx + page_size
|
||||
|
||||
return formatted_items[start_idx:end_idx], total_count
|
||||
|
||||
async def read_file(self, root: str, rel: str) -> bytes:
|
||||
"""
|
||||
读取文件内容。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
:return: 文件内容的字节流。
|
||||
"""
|
||||
api_path = self._get_api_path(rel)
|
||||
if not api_path:
|
||||
raise IsADirectoryError("不能将根目录作为文件读取")
|
||||
|
||||
resp = await self._request("GET", api_path_segment=f"{api_path}:/content")
|
||||
if resp.status_code == 404:
|
||||
raise FileNotFoundError(rel)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
|
||||
async def write_file(self, root: str, rel: str, data: bytes):
|
||||
"""
|
||||
写入文件。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
:param data: 文件内容的字节流。
|
||||
"""
|
||||
api_path = self._get_api_path(rel)
|
||||
if not api_path:
|
||||
raise ValueError("不能直接写入根路径")
|
||||
resp = await self._request("PUT", api_path_segment=f"{api_path}:/content", content=data)
|
||||
resp.raise_for_status()
|
||||
|
||||
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
|
||||
"""
|
||||
以流式方式写入文件。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
:param data_iter: 文件内容的异步迭代器。
|
||||
:return: 文件大小。
|
||||
"""
|
||||
api_path = self._get_api_path(rel)
|
||||
if not api_path:
|
||||
raise ValueError("不能直接写入根路径")
|
||||
|
||||
resp = await self._request("PUT", api_path_segment=f"{api_path}:/content", content=data_iter)
|
||||
resp.raise_for_status()
|
||||
return resp.json().get("size", 0)
|
||||
|
||||
async def mkdir(self, root: str, rel: str):
|
||||
"""
|
||||
创建目录。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
"""
|
||||
parent_path_str, new_dir_name = rel.rstrip(
|
||||
'/').rsplit('/', 1) if '/' in rel.rstrip('/') else ('', rel)
|
||||
parent_api_path = self._get_api_path(parent_path_str)
|
||||
|
||||
children_path = f"{parent_api_path}:/children" if parent_api_path else "/children"
|
||||
|
||||
payload = {
|
||||
"name": new_dir_name,
|
||||
"folder": {},
|
||||
"@microsoft.graph.conflictBehavior": "fail" # 如果已存在则失败
|
||||
}
|
||||
resp = await self._request("POST", api_path_segment=children_path, json=payload)
|
||||
resp.raise_for_status()
|
||||
|
||||
async def delete(self, root: str, rel: str):
|
||||
"""
|
||||
删除文件或目录。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
"""
|
||||
api_path = self._get_api_path(rel)
|
||||
if not api_path:
|
||||
raise ValueError("不能删除根目录")
|
||||
|
||||
resp = await self._request("DELETE", api_path_segment=api_path)
|
||||
if resp.status_code not in (204, 404):
|
||||
resp.raise_for_status()
|
||||
|
||||
async def move(self, root: str, src_rel: str, dst_rel: str):
|
||||
"""
|
||||
移动或重命名文件/目录。
|
||||
:param root: 根路径。
|
||||
:param src_rel: 源相对路径。
|
||||
:param dst_rel: 目标相对路径。
|
||||
"""
|
||||
src_api_path = self._get_api_path(src_rel)
|
||||
if not src_api_path:
|
||||
raise ValueError("不能移动根目录")
|
||||
|
||||
dst_parent_rel, dst_name = dst_rel.rstrip(
|
||||
'/').rsplit('/', 1) if '/' in dst_rel.rstrip('/') else ('', dst_rel)
|
||||
dst_parent_api_path = self._get_api_path(dst_parent_rel)
|
||||
|
||||
# 获取父项目的 ID
|
||||
parent_resp = await self._request("GET", api_path_segment=dst_parent_api_path)
|
||||
parent_resp.raise_for_status()
|
||||
parent_id = parent_resp.json()["id"]
|
||||
|
||||
payload = {
|
||||
"parentReference": {"id": parent_id},
|
||||
"name": dst_name
|
||||
}
|
||||
resp = await self._request("PATCH", api_path_segment=src_api_path, json=payload)
|
||||
resp.raise_for_status()
|
||||
|
||||
async def rename(self, root: str, src_rel: str, dst_rel: str):
|
||||
"""
|
||||
重命名文件或目录。
|
||||
在 Graph API 中,移动和重命名是同一个 PATCH 操作。
|
||||
"""
|
||||
await self.move(root, src_rel, dst_rel)
|
||||
|
||||
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
|
||||
"""
|
||||
复制文件或目录。
|
||||
:param root: 根路径。
|
||||
:param src_rel: 源相对路径。
|
||||
:param dst_rel: 目标相对路径。
|
||||
:param overwrite: 是否覆盖 (在此 API 中未直接使用)。
|
||||
"""
|
||||
src_api_path = self._get_api_path(src_rel)
|
||||
if not src_api_path:
|
||||
raise ValueError("不能复制根目录")
|
||||
|
||||
dst_parent_rel, dst_name = dst_rel.rstrip(
|
||||
'/').rsplit('/', 1) if '/' in dst_rel.rstrip('/') else ('', dst_rel)
|
||||
dst_parent_api_path = self._get_api_path(dst_parent_rel)
|
||||
|
||||
parent_resp = await self._request("GET", api_path_segment=dst_parent_api_path)
|
||||
parent_resp.raise_for_status()
|
||||
parent_id = parent_resp.json()["id"]
|
||||
|
||||
payload = {"parentReference": {"id": parent_id}, "name": dst_name}
|
||||
copy_path = f"{src_api_path}:/copy"
|
||||
resp = await self._request("POST", api_path_segment=copy_path, json=payload)
|
||||
resp.raise_for_status()
|
||||
|
||||
async def stream_file(self, root: str, rel: str, range_header: str | None):
|
||||
"""
|
||||
流式传输文件(支持范围请求)。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
:param range_header: HTTP Range 头。
|
||||
:return: FastAPI StreamingResponse 对象。
|
||||
"""
|
||||
api_path = self._get_api_path(rel)
|
||||
if not api_path:
|
||||
raise IsADirectoryError("不能对目录进行流式传输")
|
||||
|
||||
resp = await self._request("GET", api_path_segment=api_path)
|
||||
if resp.status_code == 404:
|
||||
raise FileNotFoundError(rel)
|
||||
resp.raise_for_status()
|
||||
item_data = resp.json()
|
||||
|
||||
download_url = item_data.get("@microsoft.graph.downloadUrl")
|
||||
if not download_url:
|
||||
raise Exception("无法获取下载 URL")
|
||||
|
||||
file_size = item_data.get("size", 0)
|
||||
content_type = item_data.get("file", {}).get(
|
||||
"mimeType", "application/octet-stream")
|
||||
|
||||
start = 0
|
||||
end = file_size - 1
|
||||
status = 200
|
||||
headers = {
|
||||
"Accept-Ranges": "bytes",
|
||||
"Content-Type": content_type,
|
||||
"Content-Disposition": f"inline; filename=\"{item_data.get('name')}\""
|
||||
}
|
||||
|
||||
if range_header and range_header.startswith("bytes="):
|
||||
try:
|
||||
part = range_header.removeprefix("bytes=")
|
||||
s, e = part.split("-", 1)
|
||||
if s.strip():
|
||||
start = int(s)
|
||||
if e.strip():
|
||||
end = int(e)
|
||||
if start >= file_size:
|
||||
raise HTTPException(416, "Requested Range Not Satisfiable")
|
||||
if end >= file_size:
|
||||
end = file_size - 1
|
||||
status = 206
|
||||
except ValueError:
|
||||
raise HTTPException(400, "Invalid Range header")
|
||||
|
||||
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
|
||||
headers["Content-Length"] = str(end - start + 1)
|
||||
else:
|
||||
headers["Content-Length"] = str(file_size)
|
||||
|
||||
async def file_iterator():
|
||||
nonlocal start, end
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
req_headers = {'Range': f'bytes={start}-{end}'}
|
||||
async with client.stream("GET", download_url, headers=req_headers) as stream_resp:
|
||||
stream_resp.raise_for_status()
|
||||
async for chunk in stream_resp.aiter_bytes():
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(file_iterator(), status_code=status, headers=headers, media_type=content_type)
|
||||
|
||||
async def get_direct_download_response(self, root: str, rel: str):
|
||||
if not self.enable_redirect_307:
|
||||
return None
|
||||
|
||||
api_path = self._get_api_path(rel)
|
||||
if not api_path:
|
||||
raise IsADirectoryError("不能对目录进行直链重定向")
|
||||
|
||||
resp = await self._request("GET", api_path_segment=api_path)
|
||||
if resp.status_code == 404:
|
||||
raise FileNotFoundError(rel)
|
||||
resp.raise_for_status()
|
||||
|
||||
item_data = resp.json()
|
||||
download_url = item_data.get("@microsoft.graph.downloadUrl")
|
||||
if not download_url:
|
||||
return None
|
||||
|
||||
return Response(status_code=307, headers={"Location": download_url})
|
||||
|
||||
async def get_thumbnail(self, root: str, rel: str, size: str = "medium"):
|
||||
"""
|
||||
获取文件的缩略图。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
:param size: 缩略图大小 (large, medium, small)。
|
||||
:return: 缩略图内容的字节流,或在不支持时返回 None。
|
||||
"""
|
||||
api_path = self._get_api_path(rel)
|
||||
if not api_path:
|
||||
return None
|
||||
|
||||
thumb_path = f"{api_path}:/thumbnails/0/{size}"
|
||||
|
||||
try:
|
||||
resp = await self._request("GET", api_path_segment=thumb_path)
|
||||
if resp.status_code == 200:
|
||||
thumb_data = resp.json()
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
thumb_resp = await client.get(thumb_data['url'])
|
||||
thumb_resp.raise_for_status()
|
||||
return thumb_resp.content
|
||||
elif resp.status_code == 404:
|
||||
return None
|
||||
else:
|
||||
resp.raise_for_status()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
async def stat_file(self, root: str, rel: str):
|
||||
"""
|
||||
获取文件或目录的元数据。
|
||||
:param root: 根路径。
|
||||
:param rel: 相对路径。
|
||||
:return: 格式化后的文件/目录信息。
|
||||
"""
|
||||
api_path = self._get_api_path(rel)
|
||||
resp = await self._request("GET", api_path_segment=api_path)
|
||||
if resp.status_code == 404:
|
||||
raise FileNotFoundError(rel)
|
||||
resp.raise_for_status()
|
||||
return self._format_item(resp.json())
|
||||
|
||||
|
||||
ADAPTER_TYPE = "onedrive"
|
||||
|
||||
CONFIG_SCHEMA = [
|
||||
{"key": "client_id", "label": "Client ID", "type": "string", "required": True},
|
||||
{"key": "client_secret", "label": "Client Secret",
|
||||
"type": "password", "required": True},
|
||||
{"key": "refresh_token", "label": "Refresh Token", "type": "password",
|
||||
"required": True, "help_text": "可以通过运行 'python -m domain.adapters.providers.onedrive' 获取"},
|
||||
{"key": "root", "label": "根目录 (Root Path)", "type": "string",
|
||||
"required": False, "placeholder": "默认为根目录 /"},
|
||||
{"key": "enable_direct_download_307", "label": "Enable 307 redirect download", "type": "boolean", "default": False},
|
||||
]
|
||||
|
||||
|
||||
def ADAPTER_FACTORY(rec): return OneDriveAdapter(rec)
|
||||
@@ -0,0 +1,911 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import mimetypes
|
||||
import os
|
||||
import time
|
||||
from typing import Dict, List, Tuple, Optional, AsyncIterator, Any
|
||||
|
||||
import httpx
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from models import StorageAdapter
|
||||
from .base import BaseAdapter
|
||||
|
||||
|
||||
# Quark 普通(UC)接口
|
||||
API_BASE = "https://drive.quark.cn/1/clouddrive"
|
||||
REFERER = "https://pan.quark.cn"
|
||||
PR = "ucpro"
|
||||
|
||||
|
||||
class QuarkAdapter:
|
||||
"""夸克网盘(Cookie 模式)
|
||||
|
||||
- 使用浏览器导出的 Cookie 进行鉴权
|
||||
- 通过 Quark/UC 的 clouddrive 接口实现:列目录、读写、分片上传、基础操作
|
||||
- 根 FID 固定为 "0";路径解析通过名称遍历
|
||||
"""
|
||||
|
||||
def __init__(self, record: StorageAdapter):
|
||||
self.record = record
|
||||
cfg = record.config or {}
|
||||
self.cookie: str = cfg.get("cookie") or cfg.get("Cookie")
|
||||
self.root_fid: str = cfg.get("root_fid", "0")
|
||||
def _as_bool(value: Any) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
return bool(value)
|
||||
|
||||
self.use_transcoding_address: bool = _as_bool(cfg.get("use_transcoding_address", False))
|
||||
self.only_list_video_file: bool = _as_bool(cfg.get("only_list_video_file", False))
|
||||
|
||||
if not self.cookie:
|
||||
raise ValueError("Quark 适配器需要 cookie 配置")
|
||||
|
||||
# 运行期缓存
|
||||
self._dir_fid_cache: Dict[str, str] = {f"{self.root_fid}:": self.root_fid}
|
||||
self._children_cache: Dict[str, List[Dict[str, Any]]] = {}
|
||||
|
||||
# UA 与超时
|
||||
self._ua = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) quark-cloud-drive/2.5.20 Chrome/100.0.4896.160 "
|
||||
"Electron/18.3.5.4-b478491100 Safari/537.36 Channel/pckk_other_ch"
|
||||
)
|
||||
self._timeout = 30.0
|
||||
|
||||
# -----------------
|
||||
# 工具与通用请求
|
||||
# -----------------
|
||||
def get_effective_root(self, sub_path: str | None) -> str:
|
||||
return self.root_fid
|
||||
|
||||
async def _request(
|
||||
self,
|
||||
method: str,
|
||||
pathname: str,
|
||||
*,
|
||||
json: Any | None = None,
|
||||
params: Dict[str, str] | None = None,
|
||||
) -> Any:
|
||||
headers = {
|
||||
"Cookie": self._safe_cookie(self.cookie),
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Referer": REFERER,
|
||||
"User-Agent": self._ua,
|
||||
}
|
||||
query = {"pr": PR, "fr": "pc"}
|
||||
if params:
|
||||
query.update(params)
|
||||
url = f"{API_BASE}{pathname}"
|
||||
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
resp = await client.request(method, url, headers=headers, params=query, json=json)
|
||||
# 更新运行期 cookie(若返回 __puus/__pus)
|
||||
try:
|
||||
for key in ("__puus", "__pus"):
|
||||
v = resp.cookies.get(key)
|
||||
if v:
|
||||
# 简单替换/追加到 self.cookie
|
||||
self._set_cookie_kv(key, v)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 解析业务状态
|
||||
data = None
|
||||
try:
|
||||
data = resp.json()
|
||||
except Exception:
|
||||
resp.raise_for_status()
|
||||
return resp
|
||||
status = data.get("status")
|
||||
code = data.get("code")
|
||||
msg = data.get("message") or ""
|
||||
if (status is not None and status >= 400) or (code is not None and code != 0):
|
||||
raise HTTPException(502, detail=f"Quark error status={status} code={code} msg={msg}")
|
||||
return data
|
||||
|
||||
def _set_cookie_kv(self, key: str, value: str):
|
||||
# 将指定键值写入 self.cookie(粗略字符串处理)
|
||||
parts = [p.strip() for p in (self.cookie or "").replace("\r", "").replace("\n", "").split(";") if p.strip()]
|
||||
found = False
|
||||
for i, p in enumerate(parts):
|
||||
if p.startswith(key + "="):
|
||||
parts[i] = f"{key}={value}"
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
parts.append(f"{key}={value}")
|
||||
self.cookie = "; ".join(parts)
|
||||
|
||||
def _sanitize_cookie(self, cookie: str) -> str:
|
||||
if not cookie:
|
||||
return ""
|
||||
# 去除换行与前后空白
|
||||
cookie = cookie.replace("\r", "").replace("\n", "").strip()
|
||||
# 统一分号分隔并去除多余空格/空段
|
||||
parts = [p.strip() for p in cookie.split(";") if p.strip()]
|
||||
return "; ".join(parts)
|
||||
|
||||
def _safe_cookie(self, cookie: str) -> str:
|
||||
s = self._sanitize_cookie(cookie)
|
||||
# 仅保留可见 ASCII (0x20-0x7E)
|
||||
s = "".join(ch for ch in s if 32 <= ord(ch) <= 126)
|
||||
return s
|
||||
|
||||
# -----------------
|
||||
# 列表与路径解析
|
||||
# -----------------
|
||||
def _map_file_item(self, it: Dict[str, Any]) -> Dict[str, Any]:
|
||||
# Quark/UC 列表项:file=true 表示文件;false 表示目录
|
||||
is_dir = not bool(it.get("file", False))
|
||||
updated_at_ms = int(it.get("updated_at", 0) or 0)
|
||||
name = it.get("file_name") or it.get("filename") or it.get("name")
|
||||
return {
|
||||
"fid": it.get("fid"),
|
||||
"name": name,
|
||||
"is_dir": is_dir,
|
||||
"size": 0 if is_dir else int(it.get("size", 0) or 0),
|
||||
"mtime": updated_at_ms // 1000 if updated_at_ms else 0,
|
||||
"type": "dir" if is_dir else "file",
|
||||
}
|
||||
|
||||
async def _list_children(self, parent_fid: str) -> List[Dict[str, Any]]:
|
||||
if parent_fid in self._children_cache:
|
||||
return self._children_cache[parent_fid]
|
||||
|
||||
files: List[Dict[str, Any]] = []
|
||||
page = 1
|
||||
size = 100
|
||||
total = None
|
||||
while True:
|
||||
qp = {"pdir_fid": parent_fid, "_size": str(size), "_page": str(page), "_fetch_total": "1"}
|
||||
data = await self._request("GET", "/file/sort", params=qp)
|
||||
d = (data or {}).get("data", {})
|
||||
meta = (data or {}).get("metadata", {})
|
||||
page_files = d.get("list", [])
|
||||
files.extend(page_files)
|
||||
if total is None:
|
||||
total = meta.get("_total") or meta.get("total") or 0
|
||||
if page * size >= int(total):
|
||||
break
|
||||
page += 1
|
||||
|
||||
mapped = [self._map_file_item(x) for x in files if (not self.only_list_video_file) or (not x.get("file")) or (x.get("category") == 1)]
|
||||
self._children_cache[parent_fid] = mapped
|
||||
return mapped
|
||||
|
||||
def _dir_cache_key(self, base_fid: str, rel: str) -> str:
|
||||
return f"{base_fid}:{rel.strip('/')}"
|
||||
|
||||
async def _resolve_dir_fid_from(self, base_fid: str, rel: str) -> str:
|
||||
key = rel.strip("/")
|
||||
cache_key = self._dir_cache_key(base_fid, key)
|
||||
if cache_key in self._dir_fid_cache:
|
||||
return self._dir_fid_cache[cache_key]
|
||||
if key == "":
|
||||
self._dir_fid_cache[cache_key] = base_fid
|
||||
return base_fid
|
||||
|
||||
parent_fid = base_fid
|
||||
path_so_far = []
|
||||
for seg in key.split("/"):
|
||||
if seg == "":
|
||||
continue
|
||||
path_so_far.append(seg)
|
||||
cache_key = self._dir_cache_key(base_fid, "/".join(path_so_far))
|
||||
cached = self._dir_fid_cache.get(cache_key)
|
||||
if cached:
|
||||
parent_fid = cached
|
||||
continue
|
||||
children = await self._list_children(parent_fid)
|
||||
found = next((c for c in children if c["is_dir"] and c["name"] == seg), None)
|
||||
if not found:
|
||||
raise FileNotFoundError(f"Directory not found: {seg}")
|
||||
parent_fid = found["fid"]
|
||||
self._dir_fid_cache[cache_key] = parent_fid
|
||||
|
||||
return parent_fid
|
||||
|
||||
async def _find_child(self, parent_fid: str, name: str) -> Optional[Dict[str, Any]]:
|
||||
children = await self._list_children(parent_fid)
|
||||
for it in children:
|
||||
if it["name"] == name:
|
||||
return it
|
||||
return None
|
||||
|
||||
def _invalidate_children_cache(self, parent_fid: str):
|
||||
if parent_fid in self._children_cache:
|
||||
try:
|
||||
del self._children_cache[parent_fid]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# -----------------
|
||||
# 目录与文件列表
|
||||
# -----------------
|
||||
async def list_dir(
|
||||
self,
|
||||
root: str,
|
||||
rel: str,
|
||||
page_num: int = 1,
|
||||
page_size: int = 50,
|
||||
sort_by: str = "name",
|
||||
sort_order: str = "asc",
|
||||
) -> Tuple[List[Dict], int]:
|
||||
base_fid = root or self.root_fid
|
||||
fid = await self._resolve_dir_fid_from(base_fid, rel)
|
||||
items = await self._list_children(fid)
|
||||
|
||||
# 排序,目录优先
|
||||
reverse = sort_order.lower() == "desc"
|
||||
|
||||
def get_sort_key(item):
|
||||
key = (not item["is_dir"],)
|
||||
sf = sort_by.lower()
|
||||
if sf == "name":
|
||||
key += (item["name"].lower(),)
|
||||
elif sf == "size":
|
||||
key += (item["size"],)
|
||||
elif sf == "mtime":
|
||||
key += (item["mtime"],)
|
||||
else:
|
||||
key += (item["name"].lower(),)
|
||||
return key
|
||||
|
||||
items.sort(key=get_sort_key, reverse=reverse)
|
||||
total = len(items)
|
||||
start = (page_num - 1) * page_size
|
||||
end = start + page_size
|
||||
return items[start:end], total
|
||||
|
||||
# -----------------
|
||||
# 下载与流式下载
|
||||
# -----------------
|
||||
async def _get_download_url(self, fid: str) -> str:
|
||||
data = await self._request("POST", "/file/download", json={"fids": [fid]})
|
||||
arr = (data or {}).get("data", [])
|
||||
if not arr:
|
||||
raise HTTPException(502, detail="No download data returned by Quark")
|
||||
url = arr[0].get("download_url") or arr[0].get("DownloadUrl")
|
||||
if not url:
|
||||
raise HTTPException(502, detail="No download_url returned by Quark")
|
||||
return url
|
||||
|
||||
async def _get_transcoding_url(self, fid: str) -> Optional[str]:
|
||||
try:
|
||||
payload = {"fid": fid, "resolutions": "low,normal,high,super,2k,4k", "supports": "fmp4_av,m3u8,dolby_vision"}
|
||||
data = await self._request("POST", "/file/v2/play/project", json=payload)
|
||||
lst = (data or {}).get("data", {}).get("video_list", [])
|
||||
for item in lst:
|
||||
vi = item.get("video_info") or {}
|
||||
url = vi.get("url")
|
||||
if url:
|
||||
return url
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
async def get_video_transcoding_url(self, fid: str) -> Optional[str]:
|
||||
if not self.use_transcoding_address:
|
||||
return None
|
||||
return await self._get_transcoding_url(fid)
|
||||
|
||||
def _is_video_name(self, name: str) -> bool:
|
||||
mime, _ = mimetypes.guess_type(name)
|
||||
return bool(mime and mime.startswith("video/"))
|
||||
|
||||
def _download_headers(self) -> Dict[str, str]:
|
||||
return {"Cookie": self._safe_cookie(self.cookie), "User-Agent": self._ua, "Referer": REFERER}
|
||||
|
||||
async def read_file(self, root: str, rel: str) -> bytes:
|
||||
if not rel or rel.endswith("/"):
|
||||
raise IsADirectoryError("Path is a directory")
|
||||
parent = rel.rsplit("/", 1)[0] if "/" in rel else ""
|
||||
name = rel.rsplit("/", 1)[-1]
|
||||
base_fid = root or self.root_fid
|
||||
parent_fid = await self._resolve_dir_fid_from(base_fid, parent)
|
||||
it = await self._find_child(parent_fid, name)
|
||||
if not it or it["is_dir"]:
|
||||
raise FileNotFoundError(rel)
|
||||
url = await self._get_download_url(it["fid"])
|
||||
headers = self._download_headers()
|
||||
async with httpx.AsyncClient(timeout=None, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=headers)
|
||||
if resp.status_code == 404:
|
||||
raise FileNotFoundError(rel)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
|
||||
async def read_file_range(self, root: str, rel: str, start: int, end: Optional[int] = None) -> bytes:
|
||||
if not rel or rel.endswith("/"):
|
||||
raise IsADirectoryError("Path is a directory")
|
||||
parent = rel.rsplit("/", 1)[0] if "/" in rel else ""
|
||||
name = rel.rsplit("/", 1)[-1]
|
||||
base_fid = root or self.root_fid
|
||||
parent_fid = await self._resolve_dir_fid_from(base_fid, parent)
|
||||
it = await self._find_child(parent_fid, name)
|
||||
if not it or it["is_dir"]:
|
||||
raise FileNotFoundError(rel)
|
||||
|
||||
url = await self._get_download_url(it["fid"])
|
||||
headers = dict(self._download_headers())
|
||||
headers["Range"] = f"bytes={start}-" if end is None else f"bytes={start}-{end}"
|
||||
async with httpx.AsyncClient(timeout=self._timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=headers)
|
||||
if resp.status_code == 404:
|
||||
raise FileNotFoundError(rel)
|
||||
if resp.status_code == 416:
|
||||
raise HTTPException(416, detail="Requested Range Not Satisfiable")
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
|
||||
async def stream_file(self, root: str, rel: str, range_header: str | None):
|
||||
if not rel or rel.endswith("/"):
|
||||
raise IsADirectoryError("Path is a directory")
|
||||
parent = rel.rsplit("/", 1)[0] if "/" in rel else ""
|
||||
name = rel.rsplit("/", 1)[-1]
|
||||
base_fid = root or self.root_fid
|
||||
parent_fid = await self._resolve_dir_fid_from(base_fid, parent)
|
||||
it = await self._find_child(parent_fid, name)
|
||||
if not it or it["is_dir"]:
|
||||
raise FileNotFoundError(rel)
|
||||
url = await self._get_download_url(it["fid"])
|
||||
if self.use_transcoding_address and self._is_video_name(name):
|
||||
tr = await self._get_transcoding_url(it["fid"])
|
||||
if tr:
|
||||
url = tr
|
||||
dl_headers = self._download_headers()
|
||||
|
||||
# 预获取大小/是否支持范围
|
||||
total_size: Optional[int] = None
|
||||
async with httpx.AsyncClient(timeout=self._timeout, follow_redirects=True) as client:
|
||||
try:
|
||||
head_resp = await client.head(url, headers=dl_headers)
|
||||
if head_resp.status_code == 200:
|
||||
cl = head_resp.headers.get("Content-Length")
|
||||
if cl and cl.isdigit():
|
||||
total_size = int(cl)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
mime, _ = mimetypes.guess_type(rel)
|
||||
content_type = mime or "application/octet-stream"
|
||||
|
||||
# 解析 Range
|
||||
start = 0
|
||||
end: Optional[int] = None
|
||||
status_code = 200
|
||||
if range_header and range_header.startswith("bytes="):
|
||||
status_code = 206
|
||||
part = range_header.split("=", 1)[1]
|
||||
s, e = part.split("-", 1)
|
||||
if s.strip():
|
||||
start = int(s)
|
||||
if e.strip():
|
||||
end = int(e)
|
||||
|
||||
if total_size is not None and end is None and status_code == 206:
|
||||
end = total_size - 1
|
||||
if end is not None and total_size is not None and end >= total_size:
|
||||
end = total_size - 1
|
||||
if total_size is not None and start >= total_size:
|
||||
raise HTTPException(416, detail="Requested Range Not Satisfiable")
|
||||
|
||||
resp_headers: Dict[str, str] = {"Accept-Ranges": "bytes", "Content-Type": content_type}
|
||||
if status_code == 206 and total_size is not None and end is not None:
|
||||
resp_headers["Content-Range"] = f"bytes {start}-{end}/{total_size}"
|
||||
resp_headers["Content-Length"] = str(end - start + 1)
|
||||
elif total_size is not None:
|
||||
resp_headers["Content-Length"] = str(total_size)
|
||||
|
||||
async def iterator():
|
||||
headers = dict(dl_headers)
|
||||
if status_code == 206 and end is not None:
|
||||
headers["Range"] = f"bytes={start}-{end}"
|
||||
async with httpx.AsyncClient(timeout=None, follow_redirects=True) as client:
|
||||
async with client.stream("GET", url, headers=headers) as resp:
|
||||
if resp.status_code in (404, 416):
|
||||
await resp.aclose()
|
||||
raise HTTPException(resp.status_code, detail="Upstream not available")
|
||||
async for chunk in resp.aiter_bytes():
|
||||
if chunk:
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(iterator(), status_code=status_code, headers=resp_headers, media_type=content_type)
|
||||
|
||||
# -----------------
|
||||
# 上传(大文件分片)
|
||||
# -----------------
|
||||
@staticmethod
|
||||
def _md5_hex(b: bytes) -> str:
|
||||
return hashlib.md5(b).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _sha1_hex(b: bytes) -> str:
|
||||
return hashlib.sha1(b).hexdigest()
|
||||
|
||||
def _guess_mime(self, name: str) -> str:
|
||||
mime, _ = mimetypes.guess_type(name)
|
||||
return mime or "application/octet-stream"
|
||||
|
||||
async def _upload_pre(self, filename: str, size: int, parent_fid: str) -> Dict[str, Any]:
|
||||
now_ms = int(time.time() * 1000)
|
||||
body = {
|
||||
"ccp_hash_update": True,
|
||||
"dir_name": "",
|
||||
"file_name": filename,
|
||||
"format_type": self._guess_mime(filename),
|
||||
"l_created_at": now_ms,
|
||||
"l_updated_at": now_ms,
|
||||
"pdir_fid": parent_fid,
|
||||
"size": size,
|
||||
}
|
||||
data = await self._request("POST", "/file/upload/pre", json=body)
|
||||
return data
|
||||
|
||||
async def write_file(self, root: str, rel: str, data: bytes):
|
||||
async def gen():
|
||||
yield data
|
||||
return await self.write_file_stream(root, rel, gen())
|
||||
|
||||
async def write_upload_file(self, root: str, rel: str, file_obj, filename: str | None, file_size: int | None = None, content_type: str | None = None):
|
||||
if not rel or rel.endswith("/"):
|
||||
raise HTTPException(400, detail="Invalid file path")
|
||||
|
||||
parent = rel.rsplit("/", 1)[0] if "/" in rel else ""
|
||||
name = filename or rel.rsplit("/", 1)[-1]
|
||||
base_fid = root or self.root_fid
|
||||
parent_fid = await self._resolve_dir_fid_from(base_fid, parent)
|
||||
|
||||
md5 = hashlib.md5()
|
||||
sha1 = hashlib.sha1()
|
||||
total = 0
|
||||
try:
|
||||
if callable(getattr(file_obj, "seek", None)):
|
||||
file_obj.seek(0)
|
||||
except Exception:
|
||||
pass
|
||||
while True:
|
||||
chunk = file_obj.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
total += len(chunk)
|
||||
md5.update(chunk)
|
||||
sha1.update(chunk)
|
||||
|
||||
md5_hex = md5.hexdigest()
|
||||
sha1_hex = sha1.hexdigest()
|
||||
|
||||
# 预上传,拿到上传信息
|
||||
pre_resp = await self._upload_pre(name, total, parent_fid)
|
||||
pre_data = pre_resp.get("data", {})
|
||||
|
||||
# hash 秒传
|
||||
hash_body = {"md5": md5_hex, "sha1": sha1_hex, "task_id": pre_data.get("task_id")}
|
||||
hash_resp = await self._request("POST", "/file/update/hash", json=hash_body)
|
||||
if (hash_resp.get("data") or {}).get("finish") is True:
|
||||
self._invalidate_children_cache(parent_fid)
|
||||
return {"size": total}
|
||||
|
||||
# 分片上传
|
||||
part_size = int((pre_resp.get("metadata") or {}).get("part_size") or 0)
|
||||
if part_size <= 0:
|
||||
raise HTTPException(502, detail="Invalid part_size from Quark")
|
||||
|
||||
bucket = pre_data.get("bucket")
|
||||
obj_key = pre_data.get("obj_key")
|
||||
upload_id = pre_data.get("upload_id")
|
||||
upload_url = pre_data.get("upload_url")
|
||||
if not (bucket and obj_key and upload_id and upload_url):
|
||||
raise HTTPException(502, detail="Upload pre missing fields")
|
||||
|
||||
try:
|
||||
upload_host = upload_url.split("://", 1)[1]
|
||||
except Exception:
|
||||
upload_host = upload_url
|
||||
base_url = f"https://{bucket}.{upload_host}/{obj_key}"
|
||||
|
||||
try:
|
||||
if callable(getattr(file_obj, "seek", None)):
|
||||
file_obj.seek(0)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
etags: List[str] = []
|
||||
oss_ua = "aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit"
|
||||
async with httpx.AsyncClient(timeout=None, follow_redirects=True) as client:
|
||||
part_number = 1
|
||||
left = total
|
||||
while left > 0:
|
||||
sz = min(part_size, left)
|
||||
data_bytes = file_obj.read(sz)
|
||||
if len(data_bytes) != sz:
|
||||
raise IOError("Failed to read part bytes")
|
||||
now_str = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
|
||||
auth_meta = (
|
||||
"PUT\n\n"
|
||||
f"{self._guess_mime(name)}\n"
|
||||
f"{now_str}\n"
|
||||
f"x-oss-date:{now_str}\n"
|
||||
f"x-oss-user-agent:{oss_ua}\n"
|
||||
f"/{bucket}/{obj_key}?partNumber={part_number}&uploadId={upload_id}"
|
||||
)
|
||||
auth_req_body = {"auth_info": pre_data.get("auth_info"), "auth_meta": auth_meta, "task_id": pre_data.get("task_id")}
|
||||
auth_resp = await self._request("POST", "/file/upload/auth", json=auth_req_body)
|
||||
auth_key = (auth_resp.get("data") or {}).get("auth_key")
|
||||
if not auth_key:
|
||||
raise HTTPException(502, detail="upload/auth missing auth_key")
|
||||
|
||||
put_headers = {
|
||||
"Authorization": auth_key,
|
||||
"Content-Type": self._guess_mime(name),
|
||||
"Referer": REFERER + "/",
|
||||
"x-oss-date": now_str,
|
||||
"x-oss-user-agent": oss_ua,
|
||||
}
|
||||
put_url = f"{base_url}?partNumber={part_number}&uploadId={upload_id}"
|
||||
put_resp = await client.put(put_url, headers=put_headers, content=data_bytes)
|
||||
if put_resp.status_code != 200:
|
||||
raise HTTPException(502, detail=f"Upload part failed status={put_resp.status_code} text={put_resp.text}")
|
||||
etag = put_resp.headers.get("Etag", "")
|
||||
etags.append(etag)
|
||||
left -= sz
|
||||
part_number += 1
|
||||
|
||||
parts_xml = [f"<Part>\n<PartNumber>{i+1}</PartNumber>\n<ETag>{etags[i]}</ETag>\n</Part>\n" for i in range(len(etags))]
|
||||
body_xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<CompleteMultipartUpload>\n" + "".join(parts_xml) + "</CompleteMultipartUpload>"
|
||||
content_md5 = base64.b64encode(hashlib.md5(body_xml.encode("utf-8")).digest()).decode("ascii")
|
||||
callback = pre_data.get("callback") or {}
|
||||
try:
|
||||
import json as _json
|
||||
callback_b64 = base64.b64encode(_json.dumps(callback).encode("utf-8")).decode("ascii")
|
||||
except Exception:
|
||||
callback_b64 = ""
|
||||
|
||||
now_str = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
|
||||
auth_meta_commit = (
|
||||
"POST\n"
|
||||
f"{content_md5}\n"
|
||||
"application/xml\n"
|
||||
f"{now_str}\n"
|
||||
f"x-oss-callback:{callback_b64}\n"
|
||||
f"x-oss-date:{now_str}\n"
|
||||
f"x-oss-user-agent:{oss_ua}\n"
|
||||
f"/{bucket}/{obj_key}?uploadId={upload_id}"
|
||||
)
|
||||
auth_commit_resp = await self._request("POST", "/file/upload/auth", json={"auth_info": pre_data.get("auth_info"), "auth_meta": auth_meta_commit, "task_id": pre_data.get("task_id")})
|
||||
auth_key_commit = (auth_commit_resp.get("data") or {}).get("auth_key")
|
||||
if not auth_key_commit:
|
||||
raise HTTPException(502, detail="upload/auth(commit) missing auth_key")
|
||||
|
||||
async with httpx.AsyncClient(timeout=None, follow_redirects=True) as client:
|
||||
commit_headers = {
|
||||
"Authorization": auth_key_commit,
|
||||
"Content-MD5": content_md5,
|
||||
"Content-Type": "application/xml",
|
||||
"Referer": REFERER + "/",
|
||||
"x-oss-callback": callback_b64,
|
||||
"x-oss-date": now_str,
|
||||
"x-oss-user-agent": oss_ua,
|
||||
}
|
||||
commit_url = f"{base_url}?uploadId={upload_id}"
|
||||
r = await client.post(commit_url, headers=commit_headers, content=body_xml.encode("utf-8"))
|
||||
if r.status_code != 200:
|
||||
raise HTTPException(502, detail=f"Upload commit failed status={r.status_code} text={r.text}")
|
||||
|
||||
await self._request("POST", "/file/upload/finish", json={"obj_key": obj_key, "task_id": pre_data.get("task_id")})
|
||||
try:
|
||||
await asyncio.sleep(1.0)
|
||||
except Exception:
|
||||
pass
|
||||
self._invalidate_children_cache(parent_fid)
|
||||
return {"size": total}
|
||||
|
||||
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
|
||||
if not rel or rel.endswith("/"):
|
||||
raise HTTPException(400, detail="Invalid file path")
|
||||
|
||||
parent = rel.rsplit("/", 1)[0] if "/" in rel else ""
|
||||
name = rel.rsplit("/", 1)[-1]
|
||||
base_fid = root or self.root_fid
|
||||
parent_fid = await self._resolve_dir_fid_from(base_fid, parent)
|
||||
|
||||
# 将数据落盘到临时文件,同时计算 MD5/SHA1
|
||||
import tempfile
|
||||
|
||||
md5 = hashlib.md5()
|
||||
sha1 = hashlib.sha1()
|
||||
total = 0
|
||||
with tempfile.NamedTemporaryFile(delete=False) as tf:
|
||||
tmp_path = tf.name
|
||||
try:
|
||||
async for chunk in data_iter:
|
||||
if not chunk:
|
||||
continue
|
||||
total += len(chunk)
|
||||
md5.update(chunk)
|
||||
sha1.update(chunk)
|
||||
tf.write(chunk)
|
||||
finally:
|
||||
tf.flush()
|
||||
|
||||
md5_hex = md5.hexdigest()
|
||||
sha1_hex = sha1.hexdigest()
|
||||
|
||||
# 预上传,拿到上传信息
|
||||
pre_resp = await self._upload_pre(name, total, parent_fid)
|
||||
pre_data = pre_resp.get("data", {})
|
||||
|
||||
# hash 秒传
|
||||
hash_body = {"md5": md5_hex, "sha1": sha1_hex, "task_id": pre_data.get("task_id")}
|
||||
hash_resp = await self._request("POST", "/file/update/hash", json=hash_body)
|
||||
if (hash_resp.get("data") or {}).get("finish") is True:
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except Exception:
|
||||
pass
|
||||
# 刷新父目录缓存
|
||||
self._invalidate_children_cache(parent_fid)
|
||||
return total
|
||||
|
||||
# 分片上传
|
||||
part_size = int((pre_resp.get("metadata") or {}).get("part_size") or 0)
|
||||
if part_size <= 0:
|
||||
raise HTTPException(502, detail="Invalid part_size from Quark")
|
||||
|
||||
bucket = pre_data.get("bucket")
|
||||
obj_key = pre_data.get("obj_key")
|
||||
upload_id = pre_data.get("upload_id")
|
||||
upload_url = pre_data.get("upload_url")
|
||||
if not (bucket and obj_key and upload_id and upload_url):
|
||||
raise HTTPException(502, detail="Upload pre missing fields")
|
||||
|
||||
# 计算 host 与基础 URL
|
||||
try:
|
||||
upload_host = upload_url.split("://", 1)[1]
|
||||
except Exception:
|
||||
upload_host = upload_url
|
||||
base_url = f"https://{bucket}.{upload_host}/{obj_key}"
|
||||
|
||||
# 分片循环
|
||||
etags: List[str] = []
|
||||
oss_ua = "aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit"
|
||||
async with httpx.AsyncClient(timeout=None, follow_redirects=True) as client:
|
||||
with open(tmp_path, "rb") as rf:
|
||||
part_number = 1
|
||||
left = total
|
||||
while left > 0:
|
||||
sz = min(part_size, left)
|
||||
data_bytes = rf.read(sz)
|
||||
if len(data_bytes) != sz:
|
||||
raise IOError("Failed to read part bytes")
|
||||
now_str = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
|
||||
# 申请签名
|
||||
auth_meta = (
|
||||
"PUT\n\n"
|
||||
f"{self._guess_mime(name)}\n"
|
||||
f"{now_str}\n"
|
||||
f"x-oss-date:{now_str}\n"
|
||||
f"x-oss-user-agent:{oss_ua}\n"
|
||||
f"/{bucket}/{obj_key}?partNumber={part_number}&uploadId={upload_id}"
|
||||
)
|
||||
auth_req_body = {"auth_info": pre_data.get("auth_info"), "auth_meta": auth_meta, "task_id": pre_data.get("task_id")}
|
||||
auth_resp = await self._request("POST", "/file/upload/auth", json=auth_req_body)
|
||||
auth_key = (auth_resp.get("data") or {}).get("auth_key")
|
||||
if not auth_key:
|
||||
raise HTTPException(502, detail="upload/auth missing auth_key")
|
||||
|
||||
put_headers = {
|
||||
"Authorization": auth_key,
|
||||
"Content-Type": self._guess_mime(name),
|
||||
"Referer": REFERER + "/",
|
||||
"x-oss-date": now_str,
|
||||
"x-oss-user-agent": oss_ua,
|
||||
}
|
||||
put_url = f"{base_url}?partNumber={part_number}&uploadId={upload_id}"
|
||||
put_resp = await client.put(put_url, headers=put_headers, content=data_bytes)
|
||||
if put_resp.status_code != 200:
|
||||
raise HTTPException(502, detail=f"Upload part failed status={put_resp.status_code} text={put_resp.text}")
|
||||
etag = put_resp.headers.get("Etag", "")
|
||||
etags.append(etag)
|
||||
left -= sz
|
||||
part_number += 1
|
||||
|
||||
# 组合 commit xml
|
||||
parts_xml = [f"<Part>\n<PartNumber>{i+1}</PartNumber>\n<ETag>{etags[i]}</ETag>\n</Part>\n" for i in range(len(etags))]
|
||||
body_xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<CompleteMultipartUpload>\n" + "".join(parts_xml) + "</CompleteMultipartUpload>"
|
||||
content_md5 = base64.b64encode(hashlib.md5(body_xml.encode("utf-8")).digest()).decode("ascii")
|
||||
callback = pre_data.get("callback") or {}
|
||||
try:
|
||||
import json as _json
|
||||
callback_b64 = base64.b64encode(_json.dumps(callback).encode("utf-8")).decode("ascii")
|
||||
except Exception:
|
||||
callback_b64 = ""
|
||||
|
||||
now_str = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
|
||||
auth_meta_commit = (
|
||||
"POST\n"
|
||||
f"{content_md5}\n"
|
||||
"application/xml\n"
|
||||
f"{now_str}\n"
|
||||
f"x-oss-callback:{callback_b64}\n"
|
||||
f"x-oss-date:{now_str}\n"
|
||||
f"x-oss-user-agent:{oss_ua}\n"
|
||||
f"/{bucket}/{obj_key}?uploadId={upload_id}"
|
||||
)
|
||||
auth_commit_resp = await self._request("POST", "/file/upload/auth", json={"auth_info": pre_data.get("auth_info"), "auth_meta": auth_meta_commit, "task_id": pre_data.get("task_id")})
|
||||
auth_key_commit = (auth_commit_resp.get("data") or {}).get("auth_key")
|
||||
if not auth_key_commit:
|
||||
raise HTTPException(502, detail="upload/auth(commit) missing auth_key")
|
||||
|
||||
async with httpx.AsyncClient(timeout=None, follow_redirects=True) as client:
|
||||
commit_headers = {
|
||||
"Authorization": auth_key_commit,
|
||||
"Content-MD5": content_md5,
|
||||
"Content-Type": "application/xml",
|
||||
"Referer": REFERER + "/",
|
||||
"x-oss-callback": callback_b64,
|
||||
"x-oss-date": now_str,
|
||||
"x-oss-user-agent": oss_ua,
|
||||
}
|
||||
commit_url = f"{base_url}?uploadId={upload_id}"
|
||||
r = await client.post(commit_url, headers=commit_headers, content=body_xml.encode("utf-8"))
|
||||
if r.status_code != 200:
|
||||
raise HTTPException(502, detail=f"Upload commit failed status={r.status_code} text={r.text}")
|
||||
|
||||
# finish
|
||||
await self._request("POST", "/file/upload/finish", json={"obj_key": obj_key, "task_id": pre_data.get("task_id")})
|
||||
# 端合并存在轻微延迟,等待再刷新缓存
|
||||
try:
|
||||
await asyncio.sleep(1.0)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except Exception:
|
||||
pass
|
||||
# 失效父目录缓存,确保后续列表可见
|
||||
self._invalidate_children_cache(parent_fid)
|
||||
return total
|
||||
|
||||
# -----------------
|
||||
# 基本文件操作
|
||||
# -----------------
|
||||
async def mkdir(self, root: str, rel: str):
|
||||
if not rel or rel == "/":
|
||||
raise HTTPException(400, detail="Cannot create root")
|
||||
parent = rel.rstrip("/")
|
||||
parent_rel, name = (parent.rsplit("/", 1) if "/" in parent else ("", parent))
|
||||
if not name:
|
||||
raise HTTPException(400, detail="Invalid directory name")
|
||||
pdir = await self._resolve_dir_fid_from(root or self.root_fid, parent_rel)
|
||||
await self._request("POST", "/file", json={"dir_init_lock": False, "dir_path": "", "file_name": name, "pdir_fid": pdir})
|
||||
self._invalidate_children_cache(pdir)
|
||||
|
||||
async def delete(self, root: str, rel: str):
|
||||
# 解析对象 fid + 父目录,用于失效缓存
|
||||
base_fid = root or self.root_fid
|
||||
if rel == "" or rel.endswith("/"):
|
||||
parent_rel = rel.rstrip("/")
|
||||
target_fid = await self._resolve_dir_fid_from(base_fid, parent_rel)
|
||||
parent_of_target = await self._resolve_dir_fid_from(base_fid, (parent_rel.rsplit("/", 1)[0] if "/" in parent_rel else ""))
|
||||
else:
|
||||
parent_rel, name = (rel.rsplit("/", 1) if "/" in rel else ("", rel))
|
||||
parent_of_target = await self._resolve_dir_fid_from(base_fid, parent_rel)
|
||||
it = await self._find_child(parent_of_target, name)
|
||||
if not it:
|
||||
return
|
||||
target_fid = it["fid"]
|
||||
await self._request("POST", "/file/delete", json={"action_type": 1, "exclude_fids": [], "filelist": [target_fid]})
|
||||
self._invalidate_children_cache(parent_of_target)
|
||||
|
||||
async def move(self, root: str, src_rel: str, dst_rel: str):
|
||||
# 支持跨目录与重命名:先移动到父目录,后重命名(若需要)
|
||||
src_parent_rel, src_name = (src_rel.rsplit("/", 1) if "/" in src_rel else ("", src_rel))
|
||||
dst_parent_rel, dst_name = (dst_rel.rsplit("/", 1) if "/" in dst_rel else ("", dst_rel))
|
||||
|
||||
base_fid = root or self.root_fid
|
||||
src_parent_fid = await self._resolve_dir_fid_from(base_fid, src_parent_rel)
|
||||
obj = await self._find_child(src_parent_fid, src_name)
|
||||
if not obj:
|
||||
raise FileNotFoundError(src_rel)
|
||||
dst_parent_fid = await self._resolve_dir_fid_from(base_fid, dst_parent_rel)
|
||||
|
||||
if src_parent_fid != dst_parent_fid:
|
||||
await self._request("POST", "/file/move", json={"action_type": 1, "exclude_fids": [], "filelist": [obj["fid"]], "to_pdir_fid": dst_parent_fid})
|
||||
self._invalidate_children_cache(src_parent_fid)
|
||||
self._invalidate_children_cache(dst_parent_fid)
|
||||
|
||||
if obj["name"] != dst_name:
|
||||
await self._request("POST", "/file/rename", json={"fid": obj["fid"], "file_name": dst_name})
|
||||
self._invalidate_children_cache(dst_parent_fid)
|
||||
|
||||
async def rename(self, root: str, src_rel: str, dst_rel: str):
|
||||
src_parent_rel, src_name = (src_rel.rsplit("/", 1) if "/" in src_rel else ("", src_rel))
|
||||
base_fid = root or self.root_fid
|
||||
src_parent_fid = await self._resolve_dir_fid_from(base_fid, src_parent_rel)
|
||||
obj = await self._find_child(src_parent_fid, src_name)
|
||||
if not obj:
|
||||
raise FileNotFoundError(src_rel)
|
||||
dst_name = dst_rel.rsplit("/", 1)[-1]
|
||||
await self._request("POST", "/file/rename", json={"fid": obj["fid"], "file_name": dst_name})
|
||||
self._invalidate_children_cache(src_parent_fid)
|
||||
|
||||
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
|
||||
raise NotImplementedError("QuarkOpen does not support copy via open API")
|
||||
|
||||
# -----------------
|
||||
# STAT / EXISTS / 辅助
|
||||
# -----------------
|
||||
async def stat_file(self, root: str, rel: str):
|
||||
# 通过父目录列表获取元数据
|
||||
base_fid = root or self.root_fid
|
||||
if rel == "" or rel.endswith("/"):
|
||||
# 目录
|
||||
fid = await self._resolve_dir_fid_from(base_fid, rel.rstrip("/"))
|
||||
return {"name": rel.rstrip("/").split("/")[-1] if rel else "", "is_dir": True, "size": 0, "mtime": 0, "type": "dir", "fid": fid}
|
||||
parent_rel, name = (rel.rsplit("/", 1) if "/" in rel else ("", rel))
|
||||
parent_fid = await self._resolve_dir_fid_from(base_fid, parent_rel)
|
||||
it = await self._find_child(parent_fid, name)
|
||||
if not it:
|
||||
raise FileNotFoundError(rel)
|
||||
return it
|
||||
|
||||
async def exists(self, root: str, rel: str) -> bool:
|
||||
try:
|
||||
base_fid = root or self.root_fid
|
||||
if rel == "" or rel.endswith("/"):
|
||||
await self._resolve_dir_fid_from(base_fid, rel.rstrip("/"))
|
||||
return True
|
||||
parent_rel, name = (rel.rsplit("/", 1) if "/" in rel else ("", rel))
|
||||
parent_fid = await self._resolve_dir_fid_from(base_fid, parent_rel)
|
||||
it = await self._find_child(parent_fid, name)
|
||||
return it is not None
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
async def stat_path(self, root: str, rel: str):
|
||||
# 用于 move/copy 前的预检查调试
|
||||
try:
|
||||
base_fid = root or self.root_fid
|
||||
if rel == "" or rel.endswith("/"):
|
||||
fid = await self._resolve_dir_fid_from(base_fid, rel.rstrip("/"))
|
||||
return {"exists": True, "is_dir": True, "path": rel, "fid": fid}
|
||||
parent_rel, name = (rel.rsplit("/", 1) if "/" in rel else ("", rel))
|
||||
parent_fid = await self._resolve_dir_fid_from(base_fid, parent_rel)
|
||||
it = await self._find_child(parent_fid, name)
|
||||
if it:
|
||||
return {"exists": True, "is_dir": it["is_dir"], "path": rel, "fid": it["fid"]}
|
||||
return {"exists": False, "is_dir": None, "path": rel}
|
||||
except FileNotFoundError:
|
||||
return {"exists": False, "is_dir": None, "path": rel}
|
||||
|
||||
async def _resolve_target_fid(self, rel: str, *, base_fid: Optional[str] = None) -> str:
|
||||
base = base_fid or self.root_fid
|
||||
if rel == "" or rel.endswith("/"):
|
||||
return await self._resolve_dir_fid_from(base, rel.rstrip("/"))
|
||||
parent_rel, name = (rel.rsplit("/", 1) if "/" in rel else ("", rel))
|
||||
parent_fid = await self._resolve_dir_fid_from(base, parent_rel)
|
||||
it = await self._find_child(parent_fid, name)
|
||||
if not it:
|
||||
raise FileNotFoundError(rel)
|
||||
return it["fid"]
|
||||
|
||||
|
||||
ADAPTER_TYPE = "quark"
|
||||
|
||||
CONFIG_SCHEMA = [
|
||||
{"key": "cookie", "label": "Cookie", "type": "password", "required": True, "placeholder": "从 pan.quark.cn 复制"},
|
||||
{"key": "root_fid", "label": "根 FID", "type": "string", "required": False, "default": "0"},
|
||||
{"key": "use_transcoding_address", "label": "视频转码直链", "type": "boolean", "required": False, "default": False},
|
||||
{"key": "only_list_video_file", "label": "仅列出视频文件", "type": "boolean", "required": False, "default": False},
|
||||
]
|
||||
|
||||
def ADAPTER_FACTORY(rec: StorageAdapter) -> BaseAdapter:
|
||||
return QuarkAdapter(rec)
|
||||
@@ -0,0 +1,347 @@
|
||||
import asyncio
|
||||
import mimetypes
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Tuple, AsyncIterator
|
||||
from urllib.parse import quote
|
||||
|
||||
import aioboto3
|
||||
from botocore.exceptions import ClientError
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
from models import StorageAdapter
|
||||
|
||||
|
||||
class S3Adapter:
|
||||
"""S3 兼容对象存储适配器"""
|
||||
|
||||
def __init__(self, record: StorageAdapter):
|
||||
self.record = record
|
||||
cfg = record.config
|
||||
self.bucket_name = cfg.get("bucket_name")
|
||||
self.aws_access_key_id = cfg.get("access_key_id")
|
||||
self.aws_secret_access_key = cfg.get("secret_access_key")
|
||||
self.region_name = cfg.get("region_name")
|
||||
self.endpoint_url = cfg.get("endpoint_url")
|
||||
self.root = cfg.get("root", "").strip("/")
|
||||
|
||||
if not all([self.bucket_name, self.aws_access_key_id, self.aws_secret_access_key]):
|
||||
raise ValueError(
|
||||
"S3 适配器需要 bucket_name, access_key_id, 和 secret_access_key")
|
||||
|
||||
self.session = aioboto3.Session(
|
||||
aws_access_key_id=self.aws_access_key_id,
|
||||
aws_secret_access_key=self.aws_secret_access_key,
|
||||
region_name=self.region_name,
|
||||
)
|
||||
|
||||
def get_effective_root(self, sub_path: str | None) -> str:
|
||||
"""获取 S3 中的有效根路径 (key prefix)"""
|
||||
if sub_path:
|
||||
return f"{self.root}/{sub_path.strip('/')}".strip("/")
|
||||
return self.root
|
||||
|
||||
def _get_s3_key(self, rel_path: str) -> str:
|
||||
"""将相对路径转换为 S3 key"""
|
||||
rel_path = rel_path.strip("/")
|
||||
if self.root:
|
||||
return f"{self.root}/{rel_path}"
|
||||
return rel_path
|
||||
|
||||
def _get_client(self):
|
||||
return self.session.client("s3", endpoint_url=self.endpoint_url)
|
||||
|
||||
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
|
||||
prefix = self._get_s3_key(rel)
|
||||
if prefix and not prefix.endswith("/"):
|
||||
prefix += "/"
|
||||
|
||||
all_items = []
|
||||
|
||||
async with self._get_client() as s3:
|
||||
paginator = s3.get_paginator("list_objects_v2")
|
||||
async for result in paginator.paginate(Bucket=self.bucket_name, Prefix=prefix, Delimiter="/"):
|
||||
# 添加子目录
|
||||
for common_prefix in result.get("CommonPrefixes", []):
|
||||
dir_name = common_prefix.get(
|
||||
"Prefix").removeprefix(prefix).strip("/")
|
||||
if dir_name:
|
||||
all_items.append({
|
||||
"name": dir_name,
|
||||
"is_dir": True,
|
||||
"size": 0,
|
||||
"mtime": 0,
|
||||
"type": "dir",
|
||||
})
|
||||
|
||||
# 添加文件
|
||||
for content in result.get("Contents", []):
|
||||
file_key = content.get("Key")
|
||||
if file_key == prefix: # 忽略目录本身
|
||||
continue
|
||||
file_name = file_key.removeprefix(prefix)
|
||||
if file_name:
|
||||
all_items.append({
|
||||
"name": file_name,
|
||||
"is_dir": False,
|
||||
"size": content.get("Size", 0),
|
||||
"mtime": int(content.get("LastModified", datetime.now()).timestamp()),
|
||||
"type": "file",
|
||||
})
|
||||
|
||||
# 在内存中排序和分页
|
||||
reverse = sort_order.lower() == "desc"
|
||||
def get_sort_key(item):
|
||||
key = (not item["is_dir"],)
|
||||
sort_field = sort_by.lower()
|
||||
if sort_field == "name":
|
||||
key += (item["name"].lower(),)
|
||||
elif sort_field == "size":
|
||||
key += (item["size"],)
|
||||
elif sort_field == "mtime":
|
||||
key += (item["mtime"],)
|
||||
else:
|
||||
key += (item["name"].lower(),)
|
||||
return key
|
||||
all_items.sort(key=get_sort_key, reverse=reverse)
|
||||
|
||||
total_count = len(all_items)
|
||||
start_idx = (page_num - 1) * page_size
|
||||
end_idx = start_idx + page_size
|
||||
|
||||
return all_items[start_idx:end_idx], total_count
|
||||
|
||||
async def read_file(self, root: str, rel: str) -> bytes:
|
||||
key = self._get_s3_key(rel)
|
||||
async with self._get_client() as s3:
|
||||
try:
|
||||
resp = await s3.get_object(Bucket=self.bucket_name, Key=key)
|
||||
return await resp["Body"].read()
|
||||
except ClientError as e:
|
||||
if e.response["Error"]["Code"] == "NoSuchKey":
|
||||
raise FileNotFoundError(rel)
|
||||
raise
|
||||
|
||||
async def write_file(self, root: str, rel: str, data: bytes):
|
||||
key = self._get_s3_key(rel)
|
||||
async with self._get_client() as s3:
|
||||
await s3.put_object(Bucket=self.bucket_name, Key=key, Body=data)
|
||||
|
||||
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
|
||||
key = self._get_s3_key(rel)
|
||||
MIN_PART_SIZE = 5 * 1024 * 1024
|
||||
|
||||
async with self._get_client() as s3:
|
||||
mpu = await s3.create_multipart_upload(Bucket=self.bucket_name, Key=key)
|
||||
upload_id = mpu['UploadId']
|
||||
|
||||
parts = []
|
||||
part_number = 1
|
||||
total_size = 0
|
||||
buffer = bytearray()
|
||||
|
||||
try:
|
||||
async for chunk in data_iter:
|
||||
if not chunk:
|
||||
continue
|
||||
buffer.extend(chunk)
|
||||
|
||||
while len(buffer) >= MIN_PART_SIZE:
|
||||
part_data = buffer[:MIN_PART_SIZE]
|
||||
del buffer[:MIN_PART_SIZE]
|
||||
|
||||
part = await s3.upload_part(
|
||||
Bucket=self.bucket_name,
|
||||
Key=key,
|
||||
PartNumber=part_number,
|
||||
UploadId=upload_id,
|
||||
Body=part_data
|
||||
)
|
||||
|
||||
parts.append({'PartNumber': part_number, 'ETag': part['ETag']})
|
||||
total_size += len(part_data)
|
||||
part_number += 1
|
||||
|
||||
if buffer:
|
||||
part = await s3.upload_part(
|
||||
Bucket=self.bucket_name,
|
||||
Key=key,
|
||||
PartNumber=part_number,
|
||||
UploadId=upload_id,
|
||||
Body=bytes(buffer)
|
||||
)
|
||||
parts.append({'PartNumber': part_number, 'ETag': part['ETag']})
|
||||
total_size += len(buffer)
|
||||
|
||||
await s3.complete_multipart_upload(
|
||||
Bucket=self.bucket_name,
|
||||
Key=key,
|
||||
UploadId=upload_id,
|
||||
MultipartUpload={'Parts': parts}
|
||||
)
|
||||
except Exception as e:
|
||||
await s3.abort_multipart_upload(
|
||||
Bucket=self.bucket_name,
|
||||
Key=key,
|
||||
UploadId=upload_id
|
||||
)
|
||||
raise IOError(f"S3 stream upload failed: {e}") from e
|
||||
|
||||
return total_size
|
||||
|
||||
async def mkdir(self, root: str, rel: str):
|
||||
key = self._get_s3_key(rel)
|
||||
if not key.endswith("/"):
|
||||
key += "/"
|
||||
async with self._get_client() as s3:
|
||||
await s3.put_object(Bucket=self.bucket_name, Key=key, Body=b"")
|
||||
|
||||
async def delete(self, root: str, rel: str):
|
||||
key = self._get_s3_key(rel)
|
||||
async with self._get_client() as s3:
|
||||
is_dir_like = False
|
||||
try:
|
||||
head = await s3.head_object(Bucket=self.bucket_name, Key=key)
|
||||
if head['ContentLength'] == 0 and key.endswith('/'):
|
||||
is_dir_like = True
|
||||
except ClientError as e:
|
||||
if e.response['Error']['Code'] != '404':
|
||||
raise
|
||||
|
||||
# 如果是目录,删除目录下的所有对象
|
||||
if is_dir_like or not await self.stat_file(root, rel):
|
||||
dir_key = key if key.endswith('/') else key + '/'
|
||||
paginator = s3.get_paginator("list_objects_v2")
|
||||
objects_to_delete = []
|
||||
async for result in paginator.paginate(Bucket=self.bucket_name, Prefix=dir_key):
|
||||
for content in result.get("Contents", []):
|
||||
objects_to_delete.append({"Key": content["Key"]})
|
||||
if objects_to_delete:
|
||||
await s3.delete_objects(Bucket=self.bucket_name, Delete={"Objects": objects_to_delete})
|
||||
# 如果是文件,直接删除
|
||||
else:
|
||||
await s3.delete_object(Bucket=self.bucket_name, Key=key)
|
||||
|
||||
async def move(self, root: str, src_rel: str, dst_rel: str):
|
||||
await self.copy(root, src_rel, dst_rel, overwrite=True)
|
||||
await self.delete(root, src_rel)
|
||||
|
||||
async def rename(self, root: str, src_rel: str, dst_rel: str):
|
||||
await self.move(root, src_rel, dst_rel)
|
||||
|
||||
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
|
||||
src_key = self._get_s3_key(src_rel)
|
||||
dst_key = self._get_s3_key(dst_rel)
|
||||
|
||||
async with self._get_client() as s3:
|
||||
if not overwrite:
|
||||
try:
|
||||
await s3.head_object(Bucket=self.bucket_name, Key=dst_key)
|
||||
raise FileExistsError(dst_rel)
|
||||
except ClientError as e:
|
||||
if e.response["Error"]["Code"] != "404":
|
||||
raise
|
||||
|
||||
copy_source = {"Bucket": self.bucket_name, "Key": src_key}
|
||||
await s3.copy_object(CopySource=copy_source, Bucket=self.bucket_name, Key=dst_key)
|
||||
|
||||
async def stat_file(self, root: str, rel: str):
|
||||
key = self._get_s3_key(rel)
|
||||
async with self._get_client() as s3:
|
||||
try:
|
||||
head = await s3.head_object(Bucket=self.bucket_name, Key=key)
|
||||
return {
|
||||
"name": rel.split("/")[-1],
|
||||
"is_dir": False,
|
||||
"size": head["ContentLength"],
|
||||
"mtime": int(head["LastModified"].timestamp()),
|
||||
"type": "file",
|
||||
}
|
||||
except ClientError as e:
|
||||
if e.response["Error"]["Code"] == "404":
|
||||
# 检查是否为一个 "目录"
|
||||
dir_key = key if key.endswith('/') else key + '/'
|
||||
resp = await s3.list_objects_v2(Bucket=self.bucket_name, Prefix=dir_key, MaxKeys=1)
|
||||
if resp.get('KeyCount', 0) > 0:
|
||||
return {
|
||||
"name": rel.split("/")[-1],
|
||||
"is_dir": True,
|
||||
"size": 0,
|
||||
"mtime": 0,
|
||||
"type": "dir",
|
||||
}
|
||||
raise FileNotFoundError(rel)
|
||||
raise
|
||||
|
||||
async def stream_file(self, root: str, rel: str, range_header: str | None):
|
||||
key = self._get_s3_key(rel)
|
||||
async with self._get_client() as s3:
|
||||
try:
|
||||
head = await s3.head_object(Bucket=self.bucket_name, Key=key)
|
||||
file_size = head["ContentLength"]
|
||||
content_type = head.get("ContentType", mimetypes.guess_type(key)[
|
||||
0] or "application/octet-stream")
|
||||
except ClientError as e:
|
||||
if e.response["Error"]["Code"] == "404":
|
||||
raise HTTPException(
|
||||
status_code=404, detail="File not found")
|
||||
raise
|
||||
|
||||
start = 0
|
||||
end = file_size - 1
|
||||
status = 200
|
||||
headers = {
|
||||
"Accept-Ranges": "bytes",
|
||||
"Content-Type": content_type,
|
||||
"Content-Length": str(file_size),
|
||||
"Content-Disposition": f"inline; filename=\"{quote(rel.split('/')[-1])}\""
|
||||
}
|
||||
|
||||
if range_header:
|
||||
range_val = range_header.strip().partition("=")[2]
|
||||
s, _, e = range_val.partition("-")
|
||||
try:
|
||||
start = int(s) if s else 0
|
||||
end = int(e) if e else file_size - 1
|
||||
if start >= file_size or end >= file_size or start > end:
|
||||
raise HTTPException(
|
||||
status_code=416, detail="Requested Range Not Satisfiable")
|
||||
status = 206
|
||||
headers["Content-Length"] = str(end - start + 1)
|
||||
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
|
||||
except ValueError:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Invalid Range header")
|
||||
|
||||
range_arg = f"bytes={start}-{end}"
|
||||
|
||||
async def iterator():
|
||||
try:
|
||||
resp = await s3.get_object(Bucket=self.bucket_name, Key=key, Range=range_arg)
|
||||
body = resp["Body"]
|
||||
while chunk := await body.read(65536):
|
||||
yield chunk
|
||||
except Exception as e:
|
||||
raise
|
||||
|
||||
return StreamingResponse(iterator(), status_code=status, headers=headers, media_type=content_type)
|
||||
|
||||
|
||||
ADAPTER_TYPE = "s3"
|
||||
|
||||
CONFIG_SCHEMA = [
|
||||
{"key": "bucket_name", "label": "Bucket 名称",
|
||||
"type": "string", "required": True},
|
||||
{"key": "access_key_id", "label": "Access Key ID",
|
||||
"type": "string", "required": True},
|
||||
{"key": "secret_access_key", "label": "Secret Access Key",
|
||||
"type": "password", "required": True},
|
||||
{"key": "region_name", "label": "区域 (Region)", "type": "string",
|
||||
"required": False, "placeholder": "例如 us-east-1"},
|
||||
{"key": "endpoint_url", "label": "Endpoint URL", "type": "string",
|
||||
"required": False, "placeholder": "对于 S3 兼容存储, 例如 https://minio.example.com"},
|
||||
{"key": "root", "label": "根路径 (Root Path)", "type": "string",
|
||||
"required": False, "placeholder": "在 bucket 内的路径前缀"},
|
||||
]
|
||||
|
||||
|
||||
def ADAPTER_FACTORY(rec): return S3Adapter(rec)
|
||||
@@ -0,0 +1,473 @@
|
||||
import asyncio
|
||||
import mimetypes
|
||||
import stat as statmod
|
||||
from typing import List, Dict, Tuple, AsyncIterator, Optional
|
||||
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
import paramiko
|
||||
|
||||
from models import StorageAdapter
|
||||
|
||||
|
||||
def _join_remote(root: str, rel: str) -> str:
|
||||
root = (root or "/").rstrip("/") or "/"
|
||||
rel = (rel or "").lstrip("/")
|
||||
if not rel:
|
||||
return root
|
||||
return f"{root}/{rel}"
|
||||
|
||||
|
||||
class SFTPAdapter:
|
||||
def __init__(self, record: StorageAdapter):
|
||||
self.record = record
|
||||
cfg = record.config
|
||||
self.host: str = cfg.get("host")
|
||||
self.port: int = int(cfg.get("port", 22))
|
||||
self.username: str | None = cfg.get("username")
|
||||
self.password: str | None = cfg.get("password")
|
||||
self.timeout: int = int(cfg.get("timeout", 15))
|
||||
self.root_path: str = cfg.get("root") # 必填
|
||||
self.allow_unknown_host: bool = bool(cfg.get("allow_unknown_host", True))
|
||||
|
||||
if not self.host:
|
||||
raise ValueError("SFTP adapter requires 'host'")
|
||||
if not self.username or not self.password:
|
||||
raise ValueError("SFTP adapter requires 'username' and 'password'")
|
||||
if not self.root_path:
|
||||
raise ValueError("SFTP adapter requires 'root'")
|
||||
|
||||
def get_effective_root(self, sub_path: str | None) -> str:
|
||||
base = self.root_path.rstrip("/") or "/"
|
||||
if sub_path:
|
||||
return _join_remote(base, sub_path)
|
||||
return base
|
||||
|
||||
def _connect(self) -> paramiko.SFTPClient:
|
||||
ssh = paramiko.SSHClient()
|
||||
if self.allow_unknown_host:
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
ssh.connect(
|
||||
hostname=self.host,
|
||||
port=self.port,
|
||||
username=self.username,
|
||||
password=self.password,
|
||||
timeout=self.timeout,
|
||||
allow_agent=False,
|
||||
look_for_keys=False,
|
||||
)
|
||||
return ssh.open_sftp()
|
||||
|
||||
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
|
||||
path = _join_remote(root, rel)
|
||||
|
||||
def _do_list() -> List[Dict]:
|
||||
sftp = self._connect()
|
||||
try:
|
||||
attrs = sftp.listdir_attr(path)
|
||||
entries: List[Dict] = []
|
||||
for a in attrs:
|
||||
name = a.filename
|
||||
is_dir = statmod.S_ISDIR(a.st_mode)
|
||||
entries.append({
|
||||
"name": name,
|
||||
"is_dir": is_dir,
|
||||
"size": 0 if is_dir else int(a.st_size or 0),
|
||||
"mtime": int(a.st_mtime or 0),
|
||||
"type": "dir" if is_dir else "file",
|
||||
})
|
||||
return entries
|
||||
finally:
|
||||
try:
|
||||
sftp.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
entries = await asyncio.to_thread(_do_list)
|
||||
|
||||
reverse = sort_order.lower() == "desc"
|
||||
|
||||
def get_sort_key(item):
|
||||
key = (not item["is_dir"],)
|
||||
f = sort_by.lower()
|
||||
if f == "name":
|
||||
key += (item["name"].lower(),)
|
||||
elif f == "size":
|
||||
key += (item.get("size", 0),)
|
||||
elif f == "mtime":
|
||||
key += (item.get("mtime", 0),)
|
||||
else:
|
||||
key += (item["name"].lower(),)
|
||||
return key
|
||||
|
||||
entries.sort(key=get_sort_key, reverse=reverse)
|
||||
total = len(entries)
|
||||
start = (page_num - 1) * page_size
|
||||
end = start + page_size
|
||||
return entries[start:end], total
|
||||
|
||||
async def read_file(self, root: str, rel: str) -> bytes:
|
||||
path = _join_remote(root, rel)
|
||||
|
||||
def _do_read() -> bytes:
|
||||
sftp = self._connect()
|
||||
try:
|
||||
with sftp.open(path, "rb") as f:
|
||||
return f.read()
|
||||
except FileNotFoundError:
|
||||
raise
|
||||
except IOError as e:
|
||||
if getattr(e, "errno", None) == 2:
|
||||
raise FileNotFoundError(rel)
|
||||
raise
|
||||
finally:
|
||||
try:
|
||||
sftp.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return await asyncio.to_thread(_do_read)
|
||||
|
||||
async def write_file(self, root: str, rel: str, data: bytes):
|
||||
path = _join_remote(root, rel)
|
||||
|
||||
def _ensure_dirs(sftp: paramiko.SFTPClient, dir_path: str):
|
||||
parts = [p for p in dir_path.strip("/").split("/") if p]
|
||||
cur = "/"
|
||||
for p in parts:
|
||||
cur = _join_remote(cur, p)
|
||||
try:
|
||||
sftp.mkdir(cur)
|
||||
except IOError:
|
||||
# likely exists
|
||||
pass
|
||||
|
||||
def _do_write():
|
||||
sftp = self._connect()
|
||||
try:
|
||||
parent = "/" if "/" not in path.strip("/") else path.rsplit("/", 1)[0]
|
||||
_ensure_dirs(sftp, parent)
|
||||
with sftp.open(path, "wb") as f:
|
||||
f.write(data)
|
||||
finally:
|
||||
try:
|
||||
sftp.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await asyncio.to_thread(_do_write)
|
||||
|
||||
async def write_upload_file(self, root: str, rel: str, file_obj, filename: str | None, file_size: int | None = None, content_type: str | None = None):
|
||||
path = _join_remote(root, rel)
|
||||
|
||||
def _ensure_dirs(sftp: paramiko.SFTPClient, dir_path: str):
|
||||
parts = [p for p in dir_path.strip("/").split("/") if p]
|
||||
cur = "/"
|
||||
for p in parts:
|
||||
cur = _join_remote(cur, p)
|
||||
try:
|
||||
sftp.mkdir(cur)
|
||||
except IOError:
|
||||
pass
|
||||
|
||||
def _do_upload():
|
||||
sftp = self._connect()
|
||||
try:
|
||||
parent = "/" if "/" not in path.strip("/") else path.rsplit("/", 1)[0]
|
||||
_ensure_dirs(sftp, parent)
|
||||
try:
|
||||
if callable(getattr(file_obj, "seek", None)):
|
||||
file_obj.seek(0)
|
||||
except Exception:
|
||||
pass
|
||||
with sftp.open(path, "wb") as f:
|
||||
import shutil
|
||||
shutil.copyfileobj(file_obj, f)
|
||||
finally:
|
||||
try:
|
||||
sftp.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await asyncio.to_thread(_do_upload)
|
||||
return {"size": file_size or 0}
|
||||
|
||||
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
|
||||
buf = bytearray()
|
||||
async for chunk in data_iter:
|
||||
if chunk:
|
||||
buf.extend(chunk)
|
||||
await self.write_file(root, rel, bytes(buf))
|
||||
return len(buf)
|
||||
|
||||
async def mkdir(self, root: str, rel: str):
|
||||
path = _join_remote(root, rel)
|
||||
|
||||
def _do_mkdir():
|
||||
sftp = self._connect()
|
||||
try:
|
||||
parts = [p for p in path.strip("/").split("/") if p]
|
||||
cur = "/"
|
||||
for p in parts:
|
||||
cur = _join_remote(cur, p)
|
||||
try:
|
||||
sftp.mkdir(cur)
|
||||
except IOError:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
sftp.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await asyncio.to_thread(_do_mkdir)
|
||||
|
||||
async def delete(self, root: str, rel: str):
|
||||
path = _join_remote(root, rel)
|
||||
|
||||
def _do_delete():
|
||||
sftp = self._connect()
|
||||
try:
|
||||
# Try file remove first
|
||||
try:
|
||||
sftp.remove(path)
|
||||
return
|
||||
except IOError:
|
||||
pass
|
||||
|
||||
def _rm_tree(dp: str):
|
||||
try:
|
||||
for a in sftp.listdir_attr(dp):
|
||||
child = _join_remote(dp, a.filename)
|
||||
if statmod.S_ISDIR(a.st_mode):
|
||||
_rm_tree(child)
|
||||
else:
|
||||
try:
|
||||
sftp.remove(child)
|
||||
except Exception:
|
||||
pass
|
||||
sftp.rmdir(dp)
|
||||
except IOError:
|
||||
pass
|
||||
|
||||
_rm_tree(path)
|
||||
finally:
|
||||
try:
|
||||
sftp.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await asyncio.to_thread(_do_delete)
|
||||
|
||||
async def move(self, root: str, src_rel: str, dst_rel: str):
|
||||
src = _join_remote(root, src_rel)
|
||||
dst = _join_remote(root, dst_rel)
|
||||
|
||||
def _do_move():
|
||||
sftp = self._connect()
|
||||
try:
|
||||
# ensure dst parent exists
|
||||
parent = "/" if "/" not in dst.strip("/") else dst.rsplit("/", 1)[0]
|
||||
parts = [p for p in parent.strip("/").split("/") if p]
|
||||
cur = "/"
|
||||
for p in parts:
|
||||
cur = _join_remote(cur, p)
|
||||
try:
|
||||
sftp.mkdir(cur)
|
||||
except IOError:
|
||||
pass
|
||||
sftp.rename(src, dst)
|
||||
finally:
|
||||
try:
|
||||
sftp.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await asyncio.to_thread(_do_move)
|
||||
|
||||
async def rename(self, root: str, src_rel: str, dst_rel: str):
|
||||
await self.move(root, src_rel, dst_rel)
|
||||
|
||||
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
|
||||
src = _join_remote(root, src_rel)
|
||||
dst = _join_remote(root, dst_rel)
|
||||
|
||||
def _is_dir() -> bool:
|
||||
sftp = self._connect()
|
||||
try:
|
||||
st = sftp.stat(src)
|
||||
return statmod.S_ISDIR(st.st_mode)
|
||||
finally:
|
||||
try:
|
||||
sftp.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if await asyncio.to_thread(_is_dir):
|
||||
await self.mkdir(root, dst_rel)
|
||||
|
||||
children, _ = await self.list_dir(root, src_rel, page_num=1, page_size=10_000)
|
||||
for ent in children:
|
||||
child_src = f"{src_rel.rstrip('/')}/{ent['name']}"
|
||||
child_dst = f"{dst_rel.rstrip('/')}/{ent['name']}"
|
||||
await self.copy(root, child_src, child_dst, overwrite)
|
||||
return
|
||||
|
||||
# file copy
|
||||
data = await self.read_file(root, src_rel)
|
||||
if not overwrite:
|
||||
try:
|
||||
await self.stat_file(root, dst_rel)
|
||||
raise FileExistsError(dst_rel)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
await self.write_file(root, dst_rel, data)
|
||||
|
||||
async def stat_file(self, root: str, rel: str):
|
||||
path = _join_remote(root, rel)
|
||||
|
||||
def _do_stat():
|
||||
sftp = self._connect()
|
||||
try:
|
||||
st = sftp.stat(path)
|
||||
is_dir = statmod.S_ISDIR(st.st_mode)
|
||||
info = {
|
||||
"name": rel.split("/")[-1],
|
||||
"is_dir": is_dir,
|
||||
"size": 0 if is_dir else int(st.st_size or 0),
|
||||
"mtime": int(st.st_mtime or 0),
|
||||
"type": "dir" if is_dir else "file",
|
||||
"path": path,
|
||||
}
|
||||
return info
|
||||
except FileNotFoundError:
|
||||
raise
|
||||
except IOError as e:
|
||||
if getattr(e, "errno", None) == 2:
|
||||
raise FileNotFoundError(rel)
|
||||
raise
|
||||
finally:
|
||||
try:
|
||||
sftp.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return await asyncio.to_thread(_do_stat)
|
||||
|
||||
async def exists(self, root: str, rel: str) -> bool:
|
||||
try:
|
||||
await self.stat_file(root, rel)
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def stream_file(self, root: str, rel: str, range_header: str | None):
|
||||
path = _join_remote(root, rel)
|
||||
|
||||
def _get_stat():
|
||||
sftp = self._connect()
|
||||
try:
|
||||
st = sftp.stat(path)
|
||||
return int(st.st_size or 0)
|
||||
finally:
|
||||
try:
|
||||
sftp.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
file_size = await asyncio.to_thread(_get_stat)
|
||||
if file_size is None:
|
||||
raise HTTPException(404, detail="File not found")
|
||||
|
||||
mime, _ = mimetypes.guess_type(rel)
|
||||
content_type = mime or "application/octet-stream"
|
||||
|
||||
start = 0
|
||||
end = file_size - 1
|
||||
status = 200
|
||||
headers = {
|
||||
"Accept-Ranges": "bytes",
|
||||
"Content-Type": content_type,
|
||||
"Content-Length": str(file_size),
|
||||
}
|
||||
|
||||
if range_header and range_header.startswith("bytes="):
|
||||
try:
|
||||
s, e = (range_header.removeprefix("bytes=").split("-", 1))
|
||||
if s.strip():
|
||||
start = int(s)
|
||||
if e.strip():
|
||||
end = int(e)
|
||||
if start >= file_size:
|
||||
raise HTTPException(416, detail="Requested Range Not Satisfiable")
|
||||
if end >= file_size:
|
||||
end = file_size - 1
|
||||
status = 206
|
||||
headers["Content-Length"] = str(end - start + 1)
|
||||
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
|
||||
except ValueError:
|
||||
raise HTTPException(400, detail="Invalid Range header")
|
||||
|
||||
queue: asyncio.Queue[Optional[bytes]] = asyncio.Queue(maxsize=8)
|
||||
|
||||
def _worker():
|
||||
sftp = self._connect()
|
||||
try:
|
||||
with sftp.open(path, "rb") as f:
|
||||
f.seek(start)
|
||||
remaining = end - start + 1
|
||||
chunk_size = 64 * 1024
|
||||
while remaining > 0:
|
||||
to_read = chunk_size if remaining > chunk_size else remaining
|
||||
data = f.read(to_read)
|
||||
if not data:
|
||||
break
|
||||
try:
|
||||
queue.put_nowait(data)
|
||||
except Exception:
|
||||
break
|
||||
remaining -= len(data)
|
||||
try:
|
||||
queue.put_nowait(None)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
sftp.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def agen():
|
||||
worker_fut = asyncio.to_thread(_worker)
|
||||
try:
|
||||
while True:
|
||||
chunk = await queue.get()
|
||||
if chunk is None:
|
||||
break
|
||||
yield chunk
|
||||
finally:
|
||||
try:
|
||||
await worker_fut
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return StreamingResponse(agen(), status_code=status, headers=headers, media_type=content_type)
|
||||
|
||||
|
||||
ADAPTER_TYPE = "sftp"
|
||||
|
||||
CONFIG_SCHEMA = [
|
||||
{"key": "host", "label": "主机", "type": "string", "required": True, "placeholder": "sftp.example.com"},
|
||||
{"key": "port", "label": "端口", "type": "number", "required": False, "default": 22},
|
||||
{"key": "username", "label": "用户名", "type": "string", "required": True},
|
||||
{"key": "password", "label": "密码", "type": "password", "required": True},
|
||||
{"key": "root", "label": "根路径", "type": "string", "required": True, "placeholder": "/data"},
|
||||
{"key": "timeout", "label": "超时(秒)", "type": "number", "required": False, "default": 15},
|
||||
{"key": "allow_unknown_host", "label": "允许未知主机指纹", "type": "boolean", "required": False, "default": True},
|
||||
]
|
||||
|
||||
|
||||
def ADAPTER_FACTORY(rec: StorageAdapter):
|
||||
return SFTPAdapter(rec)
|
||||
@@ -0,0 +1,578 @@
|
||||
from typing import List, Dict, Tuple, AsyncIterator
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import os
|
||||
import struct
|
||||
from models import StorageAdapter
|
||||
from telethon import TelegramClient
|
||||
from telethon.crypto import AuthKey
|
||||
from telethon.sessions import StringSession
|
||||
from telethon.tl import types
|
||||
import socks
|
||||
|
||||
_SESSION_LOCKS: Dict[str, asyncio.Lock] = {}
|
||||
|
||||
|
||||
def _get_session_lock(session_string: str) -> asyncio.Lock:
|
||||
lock = _SESSION_LOCKS.get(session_string)
|
||||
if lock is None:
|
||||
lock = asyncio.Lock()
|
||||
_SESSION_LOCKS[session_string] = lock
|
||||
return lock
|
||||
|
||||
|
||||
class _NamedFile:
|
||||
def __init__(self, file_obj, name: str):
|
||||
self._file = file_obj
|
||||
self.name = name
|
||||
|
||||
def read(self, *args, **kwargs):
|
||||
return self._file.read(*args, **kwargs)
|
||||
|
||||
def seek(self, *args, **kwargs):
|
||||
return self._file.seek(*args, **kwargs)
|
||||
|
||||
def tell(self):
|
||||
return self._file.tell()
|
||||
|
||||
def seekable(self):
|
||||
return self._file.seekable()
|
||||
|
||||
def close(self):
|
||||
return self._file.close()
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._file, name)
|
||||
|
||||
# 适配器类型标识
|
||||
ADAPTER_TYPE = "telegram"
|
||||
|
||||
# 适配器配置项定义
|
||||
CONFIG_SCHEMA = [
|
||||
{"key": "api_id", "label": "API ID", "type": "string", "required": True, "help_text": "从 my.telegram.org 获取"},
|
||||
{"key": "api_hash", "label": "API Hash", "type": "password", "required": True, "help_text": "从 my.telegram.org 获取"},
|
||||
{"key": "session_string", "label": "Session String", "type": "password", "required": True, "help_text": "通过 generate_session.py 生成"},
|
||||
{"key": "chat_id", "label": "Chat ID", "type": "string", "required": True, "placeholder": "频道/群组的ID或用户名, 例如: -100123456789 或 'channel_username'"},
|
||||
{"key": "proxy_protocol", "label": "代理协议", "type": "string", "required": False, "placeholder": "例如: socks5, http"},
|
||||
{"key": "proxy_host", "label": "代理主机", "type": "string", "required": False, "placeholder": "例如: 127.0.0.1"},
|
||||
{"key": "proxy_port", "label": "代理端口", "type": "number", "required": False, "placeholder": "例如: 1080"},
|
||||
]
|
||||
|
||||
class TelegramAdapter:
|
||||
"""Telegram 存储适配器 (使用用户 Session)"""
|
||||
|
||||
def __init__(self, record: StorageAdapter):
|
||||
self.record = record
|
||||
cfg = record.config
|
||||
self.api_id = int(cfg.get("api_id"))
|
||||
self.api_hash = cfg.get("api_hash")
|
||||
self.session_string = cfg.get("session_string")
|
||||
self.chat_id_str = cfg.get("chat_id")
|
||||
|
||||
# 代理设置
|
||||
self.proxy_protocol = cfg.get("proxy_protocol")
|
||||
self.proxy_host = cfg.get("proxy_host")
|
||||
self.proxy_port = cfg.get("proxy_port")
|
||||
|
||||
self.proxy = None
|
||||
if self.proxy_protocol and self.proxy_host and self.proxy_port:
|
||||
proto_map = {
|
||||
"socks5": socks.SOCKS5,
|
||||
"http": socks.HTTP,
|
||||
}
|
||||
proxy_type = proto_map.get(self.proxy_protocol.lower())
|
||||
if proxy_type:
|
||||
self.proxy = (proxy_type, self.proxy_host, int(self.proxy_port))
|
||||
|
||||
try:
|
||||
self.chat_id = int(self.chat_id_str)
|
||||
except (ValueError, TypeError):
|
||||
self.chat_id = self.chat_id_str
|
||||
|
||||
if not all([self.api_id, self.api_hash, self.session_string, self.chat_id]):
|
||||
raise ValueError("Telegram 适配器需要 api_id, api_hash, session_string 和 chat_id")
|
||||
|
||||
@staticmethod
|
||||
def _parse_legacy_session_string(value: str) -> StringSession:
|
||||
"""
|
||||
兼容旧版 session_string 格式:
|
||||
- version(1B char) + base64(data)
|
||||
- data: dc_id(1B) + ip_len(2B) + ip(ASCII, ip_len bytes) + port(2B) + auth_key(256B)
|
||||
"""
|
||||
s = (value or "").strip()
|
||||
if not s:
|
||||
raise ValueError("session_string 为空")
|
||||
|
||||
body = s[1:] if s.startswith("1") else s
|
||||
raw = base64.urlsafe_b64decode(body)
|
||||
if len(raw) < 1 + 2 + 2 + 256:
|
||||
raise ValueError("legacy session 数据长度不足")
|
||||
|
||||
dc_id = raw[0]
|
||||
ip_len = struct.unpack(">H", raw[1:3])[0]
|
||||
expected_len = 1 + 2 + ip_len + 2 + 256
|
||||
if len(raw) != expected_len:
|
||||
raise ValueError("legacy session 数据长度不匹配")
|
||||
|
||||
ip_start = 3
|
||||
ip_end = ip_start + ip_len
|
||||
ip = raw[ip_start:ip_end].decode("utf-8")
|
||||
port = struct.unpack(">H", raw[ip_end : ip_end + 2])[0]
|
||||
key = raw[ip_end + 2 : ip_end + 2 + 256]
|
||||
|
||||
sess = StringSession()
|
||||
sess.set_dc(dc_id, ip, port)
|
||||
sess.auth_key = AuthKey(key)
|
||||
return sess
|
||||
|
||||
@staticmethod
|
||||
def _pick_photo_thumb(thumbs: list | None):
|
||||
if not thumbs:
|
||||
return None
|
||||
|
||||
cached = []
|
||||
others = []
|
||||
for t in thumbs:
|
||||
if isinstance(t, (types.PhotoCachedSize, types.PhotoStrippedSize)):
|
||||
cached.append(t)
|
||||
elif isinstance(t, (types.PhotoSize, types.PhotoSizeProgressive)):
|
||||
if not isinstance(t, types.PhotoSizeEmpty):
|
||||
others.append(t)
|
||||
|
||||
if cached:
|
||||
cached.sort(key=lambda x: len(getattr(x, "bytes", b"") or b""))
|
||||
return cached[-1]
|
||||
|
||||
if others:
|
||||
def _sz(x):
|
||||
if isinstance(x, types.PhotoSizeProgressive):
|
||||
return max(x.sizes or [0])
|
||||
return int(getattr(x, "size", 0) or 0)
|
||||
|
||||
others.sort(key=_sz)
|
||||
return others[-1]
|
||||
|
||||
return None
|
||||
|
||||
def _build_session(self) -> StringSession:
|
||||
s = (self.session_string or "").strip()
|
||||
if not s:
|
||||
raise ValueError("Telegram 适配器 session_string 为空")
|
||||
|
||||
try:
|
||||
return StringSession(s)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 少数工具可能去掉了 version 前缀,这里做一次兼容
|
||||
if not s.startswith("1"):
|
||||
try:
|
||||
return StringSession("1" + s)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
return self._parse_legacy_session_string(s)
|
||||
except Exception as exc:
|
||||
raise ValueError("Telegram session_string 无效,请使用 Telethon StringSession 重新生成") from exc
|
||||
|
||||
def _get_client(self) -> TelegramClient:
|
||||
"""创建一个新的 TelegramClient 实例"""
|
||||
return TelegramClient(self._build_session(), self.api_id, self.api_hash, proxy=self.proxy)
|
||||
|
||||
def get_effective_root(self, sub_path: str | None) -> str:
|
||||
return ""
|
||||
|
||||
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
|
||||
if rel:
|
||||
return [], 0
|
||||
|
||||
client = self._get_client()
|
||||
entries = []
|
||||
try:
|
||||
await client.connect()
|
||||
messages = await client.get_messages(self.chat_id, limit=200)
|
||||
for message in messages:
|
||||
if not message:
|
||||
continue
|
||||
|
||||
media = message.document or message.video or message.photo
|
||||
if not media:
|
||||
continue
|
||||
|
||||
file_meta = message.file
|
||||
if not file_meta:
|
||||
continue
|
||||
|
||||
filename = file_meta.name
|
||||
if not filename:
|
||||
if message.text and '.' in message.text and len(message.text) < 256 and '\n' not in message.text:
|
||||
filename = message.text
|
||||
else:
|
||||
filename = f"unknown_{message.id}"
|
||||
|
||||
size = file_meta.size
|
||||
if size is None:
|
||||
# 兼容缺失 size 的情况
|
||||
if hasattr(media, "size") and media.size is not None:
|
||||
size = media.size
|
||||
elif message.photo and getattr(message.photo, "sizes", None):
|
||||
photo_size = message.photo.sizes[-1]
|
||||
size = getattr(photo_size, "size", 0) or 0
|
||||
else:
|
||||
size = 0
|
||||
|
||||
entries.append({
|
||||
"name": f"{message.id}_{filename}",
|
||||
"is_dir": False,
|
||||
"size": size,
|
||||
"mtime": int(message.date.timestamp()),
|
||||
"type": "file",
|
||||
})
|
||||
finally:
|
||||
if client.is_connected():
|
||||
await client.disconnect()
|
||||
|
||||
# 排序
|
||||
reverse = sort_order.lower() == "desc"
|
||||
def get_sort_key(item):
|
||||
key = (not item["is_dir"],)
|
||||
sort_field = sort_by.lower()
|
||||
if sort_field == "name":
|
||||
key += (item["name"].lower(),)
|
||||
elif sort_field == "size":
|
||||
key += (item["size"],)
|
||||
elif sort_field == "mtime":
|
||||
key += (item["mtime"],)
|
||||
else:
|
||||
key += (item["name"].lower(),)
|
||||
return key
|
||||
entries.sort(key=get_sort_key, reverse=reverse)
|
||||
|
||||
total_count = len(entries)
|
||||
|
||||
# 分页
|
||||
start_idx = (page_num - 1) * page_size
|
||||
end_idx = start_idx + page_size
|
||||
page_entries = entries[start_idx:end_idx]
|
||||
|
||||
return page_entries, total_count
|
||||
|
||||
async def read_file(self, root: str, rel: str) -> bytes:
|
||||
try:
|
||||
message_id_str, _ = rel.split('_', 1)
|
||||
message_id = int(message_id_str)
|
||||
except (ValueError, IndexError):
|
||||
raise FileNotFoundError(f"无效的文件路径格式: {rel}")
|
||||
|
||||
client = self._get_client()
|
||||
try:
|
||||
await client.connect()
|
||||
message = await client.get_messages(self.chat_id, ids=message_id)
|
||||
if not message or not (message.document or message.video or message.photo):
|
||||
raise FileNotFoundError(f"在频道 {self.chat_id} 中未找到消息ID为 {message_id} 的文件")
|
||||
|
||||
file_bytes = await client.download_media(message, file=bytes)
|
||||
return file_bytes
|
||||
finally:
|
||||
if client.is_connected():
|
||||
await client.disconnect()
|
||||
|
||||
async def write_file(self, root: str, rel: str, data: bytes):
|
||||
"""将字节数据作为文件上传"""
|
||||
client = self._get_client()
|
||||
file_like = io.BytesIO(data)
|
||||
file_like.name = os.path.basename(rel) or "file"
|
||||
|
||||
try:
|
||||
await client.connect()
|
||||
sent = await client.send_file(self.chat_id, file_like, caption=file_like.name)
|
||||
message = sent[0] if isinstance(sent, list) and sent else sent
|
||||
actual_rel = rel
|
||||
if message:
|
||||
stored_name = file_like.name
|
||||
file_meta = getattr(message, "file", None)
|
||||
if file_meta and getattr(file_meta, "name", None):
|
||||
stored_name = file_meta.name
|
||||
if getattr(message, "id", None) is not None:
|
||||
actual_rel = f"{message.id}_{stored_name}"
|
||||
return {"rel": actual_rel, "size": len(data)}
|
||||
finally:
|
||||
if client.is_connected():
|
||||
await client.disconnect()
|
||||
|
||||
async def write_upload_file(self, root: str, rel: str, file_obj, filename: str | None, file_size: int | None = None, content_type: str | None = None):
|
||||
client = self._get_client()
|
||||
name = filename or os.path.basename(rel) or "file"
|
||||
file_like = _NamedFile(file_obj, name)
|
||||
|
||||
try:
|
||||
await client.connect()
|
||||
sent = await client.send_file(
|
||||
self.chat_id,
|
||||
file_like,
|
||||
caption=file_like.name,
|
||||
file_size=file_size,
|
||||
mime_type=content_type,
|
||||
)
|
||||
message = sent[0] if isinstance(sent, list) and sent else sent
|
||||
actual_rel = rel
|
||||
size = file_size or 0
|
||||
if message:
|
||||
stored_name = file_like.name
|
||||
file_meta = getattr(message, "file", None)
|
||||
if file_meta and getattr(file_meta, "name", None):
|
||||
stored_name = file_meta.name
|
||||
if getattr(message, "id", None) is not None:
|
||||
actual_rel = f"{message.id}_{stored_name}"
|
||||
if file_meta and getattr(file_meta, "size", None):
|
||||
size = int(file_meta.size)
|
||||
return {"rel": actual_rel, "size": size}
|
||||
finally:
|
||||
if client.is_connected():
|
||||
await client.disconnect()
|
||||
|
||||
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
|
||||
"""以流式方式上传文件"""
|
||||
client = self._get_client()
|
||||
filename = os.path.basename(rel) or "file"
|
||||
import tempfile
|
||||
suffix = os.path.splitext(filename)[1]
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tf:
|
||||
temp_path = tf.name
|
||||
|
||||
total_size = 0
|
||||
try:
|
||||
with open(temp_path, "wb") as f:
|
||||
async for chunk in data_iter:
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
total_size += len(chunk)
|
||||
|
||||
await client.connect()
|
||||
sent = await client.send_file(self.chat_id, temp_path, caption=filename)
|
||||
message = sent[0] if isinstance(sent, list) and sent else sent
|
||||
actual_rel = rel
|
||||
if message:
|
||||
stored_name = filename
|
||||
file_meta = getattr(message, "file", None)
|
||||
if file_meta and getattr(file_meta, "name", None):
|
||||
stored_name = file_meta.name
|
||||
if getattr(message, "id", None) is not None:
|
||||
actual_rel = f"{message.id}_{stored_name}"
|
||||
|
||||
finally:
|
||||
if os.path.exists(temp_path):
|
||||
os.remove(temp_path)
|
||||
if client.is_connected():
|
||||
await client.disconnect()
|
||||
return {"rel": actual_rel, "size": total_size}
|
||||
|
||||
async def mkdir(self, root: str, rel: str):
|
||||
raise NotImplementedError("Telegram 适配器不支持创建目录。")
|
||||
|
||||
async def get_thumbnail(self, root: str, rel: str, size: str = "medium"):
|
||||
try:
|
||||
message_id_str, _ = rel.split('_', 1)
|
||||
message_id = int(message_id_str)
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
|
||||
client = self._get_client()
|
||||
try:
|
||||
await client.connect()
|
||||
message = await client.get_messages(self.chat_id, ids=message_id)
|
||||
if not message:
|
||||
return None
|
||||
|
||||
doc = message.document or message.video
|
||||
thumbs = None
|
||||
if doc and getattr(doc, "thumbs", None):
|
||||
thumbs = list(doc.thumbs or [])
|
||||
elif message.photo and getattr(message.photo, "sizes", None):
|
||||
thumbs = list(message.photo.sizes or [])
|
||||
|
||||
thumb = self._pick_photo_thumb(thumbs)
|
||||
if not thumb:
|
||||
return None
|
||||
|
||||
result = await client.download_media(message, bytes, thumb=thumb)
|
||||
if isinstance(result, (bytes, bytearray)):
|
||||
return bytes(result)
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
if client.is_connected():
|
||||
await client.disconnect()
|
||||
|
||||
async def delete(self, root: str, rel: str):
|
||||
"""删除一个文件 (即一条消息)"""
|
||||
try:
|
||||
message_id_str, _ = rel.split('_', 1)
|
||||
message_id = int(message_id_str)
|
||||
except (ValueError, IndexError):
|
||||
raise FileNotFoundError(f"无效的文件路径格式,无法解析消息ID: {rel}")
|
||||
|
||||
client = self._get_client()
|
||||
try:
|
||||
await client.connect()
|
||||
result = await client.delete_messages(self.chat_id, [message_id])
|
||||
if not result or not result[0].pts:
|
||||
raise FileNotFoundError(f"在 {self.chat_id} 中删除消息 {message_id} 失败,可能消息不存在或无权限")
|
||||
finally:
|
||||
if client.is_connected():
|
||||
await client.disconnect()
|
||||
|
||||
async def move(self, root: str, src_rel: str, dst_rel: str):
|
||||
raise NotImplementedError("Telegram 适配器不支持移动。")
|
||||
|
||||
async def rename(self, root: str, src_rel: str, dst_rel: str):
|
||||
raise NotImplementedError("Telegram 适配器不支持重命名。")
|
||||
|
||||
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
|
||||
raise NotImplementedError("Telegram 适配器不支持复制。")
|
||||
|
||||
async def stream_file(self, root: str, rel: str, range_header: str | None):
|
||||
from fastapi.responses import StreamingResponse
|
||||
from fastapi import HTTPException
|
||||
|
||||
try:
|
||||
message_id_str, _ = rel.split('_', 1)
|
||||
message_id = int(message_id_str)
|
||||
except (ValueError, IndexError):
|
||||
raise HTTPException(status_code=400, detail=f"无效的文件路径格式: {rel}")
|
||||
|
||||
client = self._get_client()
|
||||
lock = _get_session_lock(self.session_string)
|
||||
await lock.acquire()
|
||||
|
||||
try:
|
||||
await client.connect()
|
||||
message = await client.get_messages(self.chat_id, ids=message_id)
|
||||
media = message.document or message.video or message.photo
|
||||
if not message or not media:
|
||||
raise FileNotFoundError(f"在频道 {self.chat_id} 中未找到消息ID为 {message_id} 的文件")
|
||||
|
||||
file_meta = message.file
|
||||
file_size = file_meta.size if file_meta and file_meta.size is not None else None
|
||||
if file_size is None:
|
||||
if hasattr(media, "size") and media.size is not None:
|
||||
file_size = media.size
|
||||
elif message.photo and getattr(message.photo, "sizes", None):
|
||||
photo_size = message.photo.sizes[-1]
|
||||
file_size = getattr(photo_size, "size", 0) or 0
|
||||
else:
|
||||
file_size = 0
|
||||
|
||||
mime_type = None
|
||||
if file_meta and getattr(file_meta, "mime_type", None):
|
||||
mime_type = file_meta.mime_type
|
||||
if not mime_type:
|
||||
if hasattr(media, "mime_type") and media.mime_type:
|
||||
mime_type = media.mime_type
|
||||
elif message.photo:
|
||||
mime_type = "image/jpeg"
|
||||
else:
|
||||
mime_type = "application/octet-stream"
|
||||
|
||||
start = 0
|
||||
end = file_size - 1
|
||||
status = 200
|
||||
|
||||
headers = {
|
||||
"Accept-Ranges": "bytes",
|
||||
"Content-Type": mime_type,
|
||||
}
|
||||
|
||||
if range_header:
|
||||
try:
|
||||
range_val = range_header.strip().partition("=")[2]
|
||||
s, _, e = range_val.partition("-")
|
||||
start = int(s) if s else 0
|
||||
end = int(e) if e else file_size - 1
|
||||
if start >= file_size or end >= file_size or start > end:
|
||||
raise HTTPException(status_code=416, detail="Requested Range Not Satisfiable")
|
||||
status = 206
|
||||
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail="Invalid Range header")
|
||||
|
||||
async def iterator():
|
||||
try:
|
||||
limit = end - start + 1
|
||||
downloaded = 0
|
||||
|
||||
async for chunk in client.iter_download(media, offset=start):
|
||||
if downloaded + len(chunk) > limit:
|
||||
yield chunk[:limit - downloaded]
|
||||
break
|
||||
yield chunk
|
||||
downloaded += len(chunk)
|
||||
if downloaded >= limit:
|
||||
break
|
||||
finally:
|
||||
try:
|
||||
if client.is_connected():
|
||||
await client.disconnect()
|
||||
finally:
|
||||
lock.release()
|
||||
|
||||
return StreamingResponse(iterator(), status_code=status, headers=headers)
|
||||
|
||||
except HTTPException:
|
||||
if client.is_connected():
|
||||
await client.disconnect()
|
||||
lock.release()
|
||||
raise
|
||||
except FileNotFoundError as e:
|
||||
if client.is_connected():
|
||||
await client.disconnect()
|
||||
lock.release()
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
except Exception as e:
|
||||
if client.is_connected():
|
||||
await client.disconnect()
|
||||
lock.release()
|
||||
raise HTTPException(status_code=500, detail=f"Streaming failed: {str(e)}")
|
||||
|
||||
async def stat_file(self, root: str, rel: str):
|
||||
try:
|
||||
message_id_str, filename = rel.split('_', 1)
|
||||
message_id = int(message_id_str)
|
||||
except (ValueError, IndexError):
|
||||
raise FileNotFoundError(f"无效的文件路径格式: {rel}")
|
||||
|
||||
client = self._get_client()
|
||||
try:
|
||||
await client.connect()
|
||||
message = await client.get_messages(self.chat_id, ids=message_id)
|
||||
media = message.document or message.video or message.photo
|
||||
if not message or not media:
|
||||
raise FileNotFoundError(f"在频道 {self.chat_id} 中未找到消息ID为 {message_id} 的文件")
|
||||
|
||||
file_meta = message.file
|
||||
size = file_meta.size if file_meta and file_meta.size is not None else None
|
||||
if size is None:
|
||||
if hasattr(media, "size") and media.size is not None:
|
||||
size = media.size
|
||||
elif message.photo and getattr(message.photo, "sizes", None):
|
||||
photo_size = message.photo.sizes[-1]
|
||||
size = getattr(photo_size, "size", 0) or 0
|
||||
else:
|
||||
size = 0
|
||||
|
||||
return {
|
||||
"name": rel,
|
||||
"is_dir": False,
|
||||
"size": size,
|
||||
"mtime": int(message.date.timestamp()),
|
||||
"type": "file",
|
||||
}
|
||||
finally:
|
||||
if client.is_connected():
|
||||
await client.disconnect()
|
||||
|
||||
def ADAPTER_FACTORY(rec: StorageAdapter) -> TelegramAdapter:
|
||||
return TelegramAdapter(rec)
|
||||
@@ -0,0 +1,492 @@
|
||||
from typing import List, Dict, Optional, Tuple, AsyncIterator
|
||||
import httpx
|
||||
from urllib.parse import urljoin, quote
|
||||
from urllib.parse import urlparse, unquote
|
||||
import xml.etree.ElementTree as ET
|
||||
from models import StorageAdapter
|
||||
import mimetypes
|
||||
import logging
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import StreamingResponse, Response
|
||||
|
||||
NS = {"d": "DAV:"}
|
||||
|
||||
|
||||
class WebDAVAdapter:
|
||||
def __init__(self, record: StorageAdapter):
|
||||
self.record = record
|
||||
cfg = record.config
|
||||
self.base_url: str = cfg.get("base_url", "").rstrip('/') + '/'
|
||||
if not self.base_url.startswith("http"):
|
||||
raise ValueError("webdav requires base_url http/https")
|
||||
self.username = cfg.get("username")
|
||||
self.password = cfg.get("password")
|
||||
self.timeout = cfg.get("timeout", 15)
|
||||
|
||||
def get_effective_root(self, sub_path: str | None) -> str:
|
||||
base_url = self.record.config.get("base_url", "").rstrip('/') + '/'
|
||||
if sub_path:
|
||||
return base_url + sub_path.strip('/') + '/'
|
||||
return base_url
|
||||
|
||||
def _client(self):
|
||||
auth = (self.username, self.password) if self.username else None
|
||||
return httpx.AsyncClient(auth=auth, timeout=self.timeout, follow_redirects=True)
|
||||
|
||||
def _build_url(self, rel: str):
|
||||
rel = rel.strip('/')
|
||||
return self.base_url if not rel else urljoin(self.base_url, quote(rel) + ('/' if rel.endswith('/') else ''))
|
||||
|
||||
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
|
||||
raw_url = self._build_url(rel)
|
||||
url = raw_url if raw_url.endswith('/') else raw_url + '/'
|
||||
depth = "1"
|
||||
body = """<?xml version="1.0" encoding="utf-8" ?>
|
||||
<d:propfind xmlns:d="DAV:">
|
||||
<d:prop>
|
||||
<d:displayname />
|
||||
<d:getcontentlength />
|
||||
<d:getlastmodified />
|
||||
<d:resourcetype />
|
||||
</d:prop>
|
||||
</d:propfind>"""
|
||||
async with self._client() as client:
|
||||
resp = await client.request("PROPFIND", url, data=body, headers={"Depth": depth})
|
||||
resp.raise_for_status()
|
||||
xml_text = resp.text
|
||||
root_el = ET.fromstring(xml_text)
|
||||
all_entries: List[Dict] = []
|
||||
parsed_req = urlparse(url)
|
||||
base_path = parsed_req.path
|
||||
if not base_path.endswith('/'):
|
||||
base_path += '/'
|
||||
seen = set()
|
||||
for resp_el in root_el.findall("d:response", NS):
|
||||
href_el = resp_el.find("d:href", NS)
|
||||
if href_el is None:
|
||||
continue
|
||||
href = (href_el.text or "")
|
||||
parsed_href = urlparse(href)
|
||||
href_path = parsed_href.path or ""
|
||||
if not href_path.startswith(base_path):
|
||||
continue
|
||||
rel_path = href_path[len(base_path):].strip('/')
|
||||
if rel_path == "":
|
||||
continue
|
||||
name = unquote(rel_path.split('/')[0]).rstrip('/')
|
||||
if not name or name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
propstat = resp_el.find("d:propstat", NS)
|
||||
if propstat is None:
|
||||
continue
|
||||
prop = propstat.find("d:prop", NS)
|
||||
if prop is None:
|
||||
continue
|
||||
size_el = prop.find("d:getcontentlength", NS)
|
||||
lm_el = prop.find("d:getlastmodified", NS)
|
||||
rt_el = prop.find("d:resourcetype", NS)
|
||||
is_dir = rt_el.find(
|
||||
"d:collection", NS) is not None if rt_el is not None else href_path.endswith('/')
|
||||
size = int(
|
||||
size_el.text) if size_el is not None and size_el.text and size_el.text.isdigit() else 0
|
||||
|
||||
from email.utils import parsedate_to_datetime
|
||||
mtime = 0
|
||||
if lm_el is not None and lm_el.text:
|
||||
try:
|
||||
mtime = int(parsedate_to_datetime(lm_el.text).timestamp())
|
||||
except Exception:
|
||||
mtime = 0
|
||||
|
||||
all_entries.append({
|
||||
"name": name,
|
||||
"is_dir": is_dir,
|
||||
"size": 0 if is_dir else size,
|
||||
"mtime": mtime,
|
||||
"type": "dir" if is_dir else "file",
|
||||
})
|
||||
|
||||
# 排序所有条目
|
||||
reverse = sort_order.lower() == "desc"
|
||||
def get_sort_key(item):
|
||||
key = (not item["is_dir"],)
|
||||
sort_field = sort_by.lower()
|
||||
if sort_field == "name":
|
||||
key += (item["name"].lower(),)
|
||||
elif sort_field == "size":
|
||||
key += (item["size"],)
|
||||
elif sort_field == "mtime":
|
||||
key += (item["mtime"],)
|
||||
else:
|
||||
key += (item["name"].lower(),)
|
||||
return key
|
||||
all_entries.sort(key=get_sort_key, reverse=reverse)
|
||||
|
||||
total_count = len(all_entries)
|
||||
|
||||
# 应用分页
|
||||
start_idx = (page_num - 1) * page_size
|
||||
end_idx = start_idx + page_size
|
||||
page_entries = all_entries[start_idx:end_idx]
|
||||
|
||||
return page_entries, total_count
|
||||
|
||||
async def read_file(self, root: str, rel: str) -> bytes:
|
||||
url = self._build_url(rel)
|
||||
async with self._client() as client:
|
||||
resp = await client.get(url)
|
||||
if resp.status_code == 404:
|
||||
raise FileNotFoundError(rel)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
|
||||
async def write_file(self, root: str, rel: str, data: bytes):
|
||||
url = self._build_url(rel)
|
||||
async with self._client() as client:
|
||||
resp = await client.put(url, content=data)
|
||||
resp.raise_for_status()
|
||||
|
||||
async def mkdir(self, root: str, rel: str):
|
||||
url = self._build_url(rel.rstrip('/') + '/')
|
||||
async with self._client() as client:
|
||||
resp = await client.request("MKCOL", url)
|
||||
if resp.status_code not in (201, 405):
|
||||
resp.raise_for_status()
|
||||
|
||||
async def delete(self, root: str, rel: str):
|
||||
url = self._build_url(rel)
|
||||
async with self._client() as client:
|
||||
resp = await client.delete(url)
|
||||
if resp.status_code not in (204, 200, 404):
|
||||
resp.raise_for_status()
|
||||
|
||||
async def move(self, root: str, src_rel: str, dst_rel: str):
|
||||
src_url = self._build_url(src_rel)
|
||||
dst_url = self._build_url(dst_rel)
|
||||
async with self._client() as client:
|
||||
resp = await client.request("MOVE", src_url, headers={"Destination": dst_url})
|
||||
resp.raise_for_status()
|
||||
|
||||
async def rename(self, root: str, src_rel: str, dst_rel: str):
|
||||
src_url = self._build_url(src_rel)
|
||||
dst_url = self._build_url(dst_rel)
|
||||
async with self._client() as client:
|
||||
resp = await client.request("MOVE", src_url, headers={"Destination": dst_url})
|
||||
resp.raise_for_status()
|
||||
|
||||
async def get_file_size(self, root: str, rel: str) -> int:
|
||||
"""获取文件大小"""
|
||||
url = self._build_url(rel)
|
||||
async with self._client() as client:
|
||||
# 使用HEAD请求获取文件信息
|
||||
resp = await client.head(url)
|
||||
if resp.status_code == 404:
|
||||
raise FileNotFoundError(rel)
|
||||
resp.raise_for_status()
|
||||
|
||||
content_length = resp.headers.get('content-length')
|
||||
if content_length:
|
||||
return int(content_length)
|
||||
|
||||
# 如果HEAD不返回content-length,尝试PROPFIND
|
||||
body = """<?xml version="1.0" encoding="utf-8" ?>
|
||||
<d:propfind xmlns:d="DAV:">
|
||||
<d:prop>
|
||||
<d:getcontentlength />
|
||||
</d:prop>
|
||||
</d:propfind>"""
|
||||
resp = await client.request("PROPFIND", url, data=body, headers={"Depth": "0"})
|
||||
resp.raise_for_status()
|
||||
|
||||
root_el = ET.fromstring(resp.text)
|
||||
for resp_el in root_el.findall("d:response", NS):
|
||||
propstat = resp_el.find("d:propstat", NS)
|
||||
if propstat is None:
|
||||
continue
|
||||
prop = propstat.find("d:prop", NS)
|
||||
if prop is None:
|
||||
continue
|
||||
size_el = prop.find("d:getcontentlength", NS)
|
||||
if size_el is not None and size_el.text and size_el.text.isdigit():
|
||||
return int(size_el.text)
|
||||
|
||||
return 0
|
||||
|
||||
async def read_file_range(self, root: str, rel: str, start: int, end: Optional[int] = None) -> bytes:
|
||||
"""读取文件的指定范围"""
|
||||
url = self._build_url(rel)
|
||||
|
||||
# 构建Range头
|
||||
if end is None:
|
||||
range_header = f"bytes={start}-"
|
||||
else:
|
||||
range_header = f"bytes={start}-{end}"
|
||||
|
||||
async with self._client() as client:
|
||||
resp = await client.get(url, headers={"Range": range_header})
|
||||
if resp.status_code == 404:
|
||||
raise FileNotFoundError(rel)
|
||||
if resp.status_code not in (200, 206): # 206是Partial Content
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
|
||||
async def stream_file(self, root: str, rel: str, range_header: str | None):
|
||||
url = self._build_url(rel)
|
||||
mime, _ = mimetypes.guess_type(rel)
|
||||
content_type = mime or "application/octet-stream"
|
||||
logger = logging.getLogger(__name__)
|
||||
timeout = self.timeout
|
||||
auth = (self.username, self.password) if self.username else None
|
||||
|
||||
client_start = 0
|
||||
client_end = None
|
||||
status_code = 200
|
||||
if range_header and range_header.startswith("bytes="):
|
||||
status_code = 206
|
||||
part = range_header.removeprefix("bytes=")
|
||||
s, e = part.split("-", 1)
|
||||
if s.strip():
|
||||
client_start = int(s)
|
||||
if e.strip():
|
||||
client_end = int(e)
|
||||
|
||||
total_size = None
|
||||
accept_ranges = False
|
||||
async with httpx.AsyncClient(timeout=timeout, auth=auth, follow_redirects=True) as client:
|
||||
try:
|
||||
head_resp = await client.head(url)
|
||||
if head_resp.status_code == 404:
|
||||
raise HTTPException(404, detail="File not found")
|
||||
if head_resp.status_code == 200:
|
||||
cl = head_resp.headers.get("Content-Length")
|
||||
if cl and cl.isdigit():
|
||||
total_size = int(cl)
|
||||
ar = head_resp.headers.get("Accept-Ranges", "").lower()
|
||||
accept_ranges = "bytes" in ar
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.debug("HEAD failed %s err=%s", url, e)
|
||||
if total_size is None and (client_end is None):
|
||||
try:
|
||||
probe_req = client.build_request("GET", url, headers={"Range": "bytes=0-0"})
|
||||
probe_resp = await client.send(probe_req, stream=True)
|
||||
if probe_resp.status_code in (200, 206):
|
||||
cr = probe_resp.headers.get("Content-Range")
|
||||
if cr and "/" in cr:
|
||||
try:
|
||||
total_size = int(cr.rsplit("/", 1)[1])
|
||||
except Exception:
|
||||
pass
|
||||
await probe_resp.aclose()
|
||||
except Exception as e:
|
||||
logger.debug("Probe 0-0 failed %s err=%s", url, e)
|
||||
|
||||
if total_size is not None and client_end is None:
|
||||
client_end = total_size - 1
|
||||
if client_end is not None and client_end < client_start:
|
||||
raise HTTPException(416, detail="Requested Range Not Satisfiable")
|
||||
|
||||
# 若客户端未请求范围且上游不支持 Range,直接透传
|
||||
if status_code == 200 and (range_header is None) and not accept_ranges:
|
||||
async with httpx.AsyncClient(timeout=timeout, auth=auth, follow_redirects=True) as client:
|
||||
req = client.build_request("GET", url)
|
||||
resp = await client.send(req, stream=True)
|
||||
if resp.status_code == 404:
|
||||
await resp.aclose()
|
||||
raise HTTPException(404, detail="File not found")
|
||||
upstream_ct = resp.headers.get("Content-Type", content_type)
|
||||
|
||||
async def passthrough():
|
||||
try:
|
||||
async for chunk in resp.aiter_bytes():
|
||||
if chunk:
|
||||
yield chunk
|
||||
finally:
|
||||
await resp.aclose()
|
||||
return StreamingResponse(passthrough(), status_code=resp.status_code,
|
||||
headers={"Accept-Ranges": "bytes",
|
||||
"X-VFS-Remote-Status": str(resp.status_code)},
|
||||
media_type=upstream_ct)
|
||||
|
||||
SEGMENT_SIZE = 5 * 1024 * 1024
|
||||
MAX_RETRY_PER_SEG = 3
|
||||
FIRST_BYTE_MAX_RETRY = 3
|
||||
|
||||
resp_headers = {
|
||||
"Accept-Ranges": "bytes",
|
||||
"Content-Type": content_type,
|
||||
"X-VFS-Segmented": "1",
|
||||
}
|
||||
if status_code == 206 and total_size is not None:
|
||||
resp_headers["Content-Range"] = f"bytes {client_start}-{client_end}/{total_size}"
|
||||
|
||||
async def segmented_body():
|
||||
current = client_start
|
||||
first_byte_sent = False
|
||||
while True:
|
||||
if client_end is not None and current > client_end:
|
||||
break
|
||||
seg_start = current
|
||||
seg_end = (min(seg_start + SEGMENT_SIZE - 1, client_end)
|
||||
if client_end is not None else seg_start + SEGMENT_SIZE - 1)
|
||||
attempt = 0
|
||||
ok = False
|
||||
while attempt < MAX_RETRY_PER_SEG and not ok:
|
||||
attempt += 1
|
||||
headers_req = {"Range": f"bytes={seg_start}-{seg_end}"}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout, auth=auth, follow_redirects=True) as cseg:
|
||||
req = cseg.build_request("GET", url, headers=headers_req)
|
||||
rseg = await cseg.send(req, stream=True)
|
||||
if rseg.status_code in (200, 206):
|
||||
async for chunk in rseg.aiter_bytes():
|
||||
if chunk:
|
||||
first_byte_sent = True
|
||||
yield chunk
|
||||
await rseg.aclose()
|
||||
ok = True
|
||||
elif rseg.status_code == 404:
|
||||
await rseg.aclose()
|
||||
if not first_byte_sent:
|
||||
raise HTTPException(404, detail="File not found")
|
||||
return
|
||||
else:
|
||||
await rseg.aclose()
|
||||
logger.warning("Segment unexpected status %s %s-%s %s", rel, seg_start, seg_end, rseg.status_code)
|
||||
if not ok:
|
||||
continue
|
||||
except (httpx.ReadError, httpx.HTTPError, httpx.StreamError) as e:
|
||||
if not first_byte_sent and attempt >= FIRST_BYTE_MAX_RETRY:
|
||||
raise HTTPException(502, detail=f"Upstream error before first byte err={e}")
|
||||
logger.warning("Segment error %s %s-%s attempt=%d err=%s", rel, seg_start, seg_end, attempt, e)
|
||||
except Exception as e:
|
||||
if not first_byte_sent:
|
||||
raise
|
||||
logger.error("Segment unexpected %s %s-%s attempt=%d err=%s", rel, seg_start, seg_end, attempt, e)
|
||||
if not ok:
|
||||
logger.error("Abort streaming %s at %s-%s", rel, seg_start, seg_end)
|
||||
break
|
||||
current = seg_end + 1
|
||||
if client_end is None:
|
||||
continue
|
||||
if current > client_end:
|
||||
break
|
||||
|
||||
return StreamingResponse(segmented_body(), status_code=status_code, headers=resp_headers, media_type=content_type)
|
||||
|
||||
async def stat_file(self, root: str, rel: str):
|
||||
url = self._build_url(rel)
|
||||
async with self._client() as client:
|
||||
# PROPFIND 获取属性
|
||||
body = """<?xml version="1.0" encoding="utf-8" ?>
|
||||
<d:propfind xmlns:d="DAV:">
|
||||
<d:prop>
|
||||
<d:getcontentlength />
|
||||
<d:getlastmodified />
|
||||
<d:resourcetype />
|
||||
</d:prop>
|
||||
</d:propfind>"""
|
||||
resp = await client.request("PROPFIND", url, data=body, headers={"Depth": "0"})
|
||||
if resp.status_code == 404:
|
||||
raise FileNotFoundError(rel)
|
||||
resp.raise_for_status()
|
||||
root_el = ET.fromstring(resp.text)
|
||||
info = {
|
||||
"name": rel.split("/")[-1],
|
||||
"is_dir": False,
|
||||
"size": None,
|
||||
"mtime": None,
|
||||
"type": "file",
|
||||
"path": url,
|
||||
}
|
||||
for resp_el in root_el.findall("d:response", NS):
|
||||
propstat = resp_el.find("d:propstat", NS)
|
||||
if propstat is None:
|
||||
continue
|
||||
prop = propstat.find("d:prop", NS)
|
||||
if prop is None:
|
||||
continue
|
||||
size_el = prop.find("d:getcontentlength", NS)
|
||||
lm_el = prop.find("d:getlastmodified", NS)
|
||||
rt_el = prop.find("d:resourcetype", NS)
|
||||
is_dir = rt_el.find("d:collection", NS) is not None if rt_el is not None else False
|
||||
info["is_dir"] = is_dir
|
||||
info["type"] = "dir" if is_dir else "file"
|
||||
if size_el is not None and size_el.text and size_el.text.isdigit():
|
||||
info["size"] = int(size_el.text)
|
||||
elif info["size"] is None:
|
||||
info["size"] = 0
|
||||
if lm_el is not None and lm_el.text:
|
||||
from email.utils import parsedate_to_datetime
|
||||
try:
|
||||
info["mtime"] = int(parsedate_to_datetime(lm_el.text).timestamp())
|
||||
except Exception:
|
||||
info["mtime"] = 0
|
||||
elif info["mtime"] is None:
|
||||
info["mtime"] = 0
|
||||
# exif信息
|
||||
exif = None
|
||||
if not info["is_dir"]:
|
||||
mime, _ = mimetypes.guess_type(info["name"])
|
||||
if mime and mime.startswith("image/"):
|
||||
try:
|
||||
resp_img = await client.get(url)
|
||||
if resp_img.status_code == 200:
|
||||
from PIL import Image
|
||||
from io import BytesIO
|
||||
img = Image.open(BytesIO(resp_img.content))
|
||||
exif_data = img._getexif()
|
||||
if exif_data:
|
||||
exif = {str(k): str(v) for k, v in exif_data.items()}
|
||||
except Exception:
|
||||
exif = None
|
||||
info["exif"] = exif
|
||||
return info
|
||||
|
||||
async def exists(self, root: str, rel: str) -> bool:
|
||||
url = self._build_url(rel)
|
||||
async with self._client() as client:
|
||||
try:
|
||||
r = await client.head(url)
|
||||
return r.status_code in (200, 204)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
|
||||
url = self._build_url(rel)
|
||||
async def agen():
|
||||
async for chunk in data_iter:
|
||||
if chunk:
|
||||
yield chunk
|
||||
async with self._client() as client:
|
||||
resp = await client.put(url, content=agen())
|
||||
resp.raise_for_status()
|
||||
return True
|
||||
|
||||
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
|
||||
src_url = self._build_url(src_rel)
|
||||
dst_url = self._build_url(dst_rel)
|
||||
headers = {
|
||||
"Destination": dst_url,
|
||||
"Overwrite": "T" if overwrite else "F"
|
||||
}
|
||||
async with self._client() as client:
|
||||
resp = await client.request("COPY", src_url, headers=headers)
|
||||
if resp.status_code == 412:
|
||||
raise FileExistsError(dst_rel)
|
||||
if resp.status_code == 404:
|
||||
raise FileNotFoundError(src_rel)
|
||||
resp.raise_for_status()
|
||||
|
||||
ADAPTER_TYPE = "webdav"
|
||||
CONFIG_SCHEMA = [
|
||||
{"key": "base_url", "label": "基础地址", "type": "string",
|
||||
"required": True, "placeholder": "https://example.com/dav/"},
|
||||
{"key": "username", "label": "用户名", "type": "string", "required": False},
|
||||
{"key": "password", "label": "密码", "type": "password", "required": False},
|
||||
{"key": "timeout",
|
||||
"label": "超时(秒)", "type": "number", "required": False, "default": 15},
|
||||
]
|
||||
def ADAPTER_FACTORY(rec): return WebDAVAdapter(rec)
|
||||
@@ -0,0 +1,157 @@
|
||||
import inspect
|
||||
import pkgutil
|
||||
from importlib import import_module
|
||||
from typing import Callable, Dict
|
||||
|
||||
from models import StorageAdapter
|
||||
from .providers.base import BaseAdapter
|
||||
|
||||
AdapterFactory = Callable[[StorageAdapter], BaseAdapter]
|
||||
|
||||
TYPE_MAP: Dict[str, AdapterFactory] = {}
|
||||
CONFIG_SCHEMAS: Dict[str, list] = {}
|
||||
|
||||
|
||||
def normalize_adapter_type(value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = str(value).strip().lower()
|
||||
return normalized or None
|
||||
|
||||
|
||||
def discover_adapters():
|
||||
"""扫描 domain.adapters.providers 包, 自动注册适配器类型、工厂与配置 schema。"""
|
||||
from . import providers as adapters_pkg
|
||||
|
||||
TYPE_MAP.clear()
|
||||
CONFIG_SCHEMAS.clear()
|
||||
for modinfo in pkgutil.iter_modules(adapters_pkg.__path__):
|
||||
if modinfo.name.startswith("_"):
|
||||
continue
|
||||
full_name = f"{adapters_pkg.__name__}.{modinfo.name}"
|
||||
try:
|
||||
module = import_module(full_name)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
adapter_types = getattr(module, "ADAPTER_TYPES", None)
|
||||
if isinstance(adapter_types, dict):
|
||||
default_schema = getattr(module, "CONFIG_SCHEMA", None)
|
||||
schema_map = getattr(module, "CONFIG_SCHEMA_MAP", None)
|
||||
if not isinstance(schema_map, dict):
|
||||
schema_map = None
|
||||
|
||||
for adapter_type, factory in adapter_types.items():
|
||||
normalized_type = normalize_adapter_type(adapter_type)
|
||||
if not normalized_type:
|
||||
continue
|
||||
if not callable(factory):
|
||||
continue
|
||||
TYPE_MAP[normalized_type] = factory
|
||||
|
||||
schema = schema_map.get(normalized_type) if schema_map else default_schema
|
||||
if isinstance(schema, list):
|
||||
CONFIG_SCHEMAS[normalized_type] = schema
|
||||
continue
|
||||
|
||||
adapter_type = normalize_adapter_type(getattr(module, "ADAPTER_TYPE", None))
|
||||
schema = getattr(module, "CONFIG_SCHEMA", None)
|
||||
factory = getattr(module, "ADAPTER_FACTORY", None)
|
||||
|
||||
if not adapter_type:
|
||||
continue
|
||||
|
||||
if factory is None:
|
||||
for attr in module.__dict__.values():
|
||||
if inspect.isclass(attr) and attr.__name__.endswith("Adapter"):
|
||||
def _mk(cls=attr):
|
||||
return lambda rec: cls(rec)
|
||||
factory = _mk()
|
||||
break
|
||||
if not callable(factory):
|
||||
continue
|
||||
|
||||
TYPE_MAP[adapter_type] = factory
|
||||
if isinstance(schema, list):
|
||||
CONFIG_SCHEMAS[adapter_type] = schema
|
||||
|
||||
|
||||
def get_config_schemas() -> Dict[str, list]:
|
||||
return CONFIG_SCHEMAS
|
||||
|
||||
|
||||
def get_config_schema(adapter_type: str):
|
||||
return CONFIG_SCHEMAS.get(adapter_type)
|
||||
|
||||
|
||||
class RuntimeRegistry:
|
||||
def __init__(self):
|
||||
self._instances: Dict[int, BaseAdapter] = {}
|
||||
|
||||
async def refresh(self):
|
||||
discover_adapters()
|
||||
self._instances.clear()
|
||||
adapters = await StorageAdapter.filter(enabled=True)
|
||||
for rec in adapters:
|
||||
normalized_type = normalize_adapter_type(rec.type)
|
||||
if not normalized_type:
|
||||
continue
|
||||
if normalized_type != rec.type:
|
||||
rec.type = normalized_type
|
||||
try:
|
||||
await rec.save(update_fields=["type"])
|
||||
except Exception:
|
||||
continue
|
||||
factory = TYPE_MAP.get(normalized_type)
|
||||
if not factory:
|
||||
continue
|
||||
try:
|
||||
self._instances[rec.id] = factory(rec)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
def get(self, adapter_id: int) -> BaseAdapter | None:
|
||||
return self._instances.get(adapter_id)
|
||||
|
||||
def snapshot(self) -> Dict[int, BaseAdapter]:
|
||||
return dict(self._instances)
|
||||
|
||||
def remove(self, adapter_id: int):
|
||||
"""从缓存中移除一个适配器实例"""
|
||||
if adapter_id in self._instances:
|
||||
del self._instances[adapter_id]
|
||||
|
||||
async def upsert(self, rec: StorageAdapter):
|
||||
"""新增或更新一个适配器实例"""
|
||||
if not rec.enabled:
|
||||
self.remove(rec.id)
|
||||
return
|
||||
|
||||
normalized_type = normalize_adapter_type(rec.type)
|
||||
if not normalized_type:
|
||||
self.remove(rec.id)
|
||||
return
|
||||
if normalized_type != rec.type:
|
||||
rec.type = normalized_type
|
||||
try:
|
||||
await rec.save(update_fields=["type"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
factory = TYPE_MAP.get(normalized_type)
|
||||
if not factory:
|
||||
discover_adapters()
|
||||
factory = TYPE_MAP.get(normalized_type)
|
||||
if not factory:
|
||||
return
|
||||
|
||||
try:
|
||||
instance = factory(rec)
|
||||
self._instances[rec.id] = instance
|
||||
except Exception:
|
||||
self.remove(rec.id)
|
||||
pass
|
||||
|
||||
|
||||
runtime_registry = RuntimeRegistry()
|
||||
discover_adapters()
|
||||
@@ -0,0 +1,116 @@
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from domain.auth import User
|
||||
from .registry import (
|
||||
get_config_schemas,
|
||||
normalize_adapter_type,
|
||||
runtime_registry,
|
||||
)
|
||||
from .types import AdapterCreate, AdapterOut
|
||||
from models import StorageAdapter
|
||||
|
||||
|
||||
class AdapterService:
|
||||
@classmethod
|
||||
def _validate_and_normalize_config(cls, adapter_type: str, cfg):
|
||||
schemas = get_config_schemas()
|
||||
adapter_type = normalize_adapter_type(adapter_type)
|
||||
if not adapter_type:
|
||||
raise HTTPException(400, detail="不支持的适配器类型")
|
||||
if not isinstance(cfg, dict):
|
||||
raise HTTPException(400, detail="config 必须是对象")
|
||||
schema = schemas.get(adapter_type)
|
||||
if not schema:
|
||||
raise HTTPException(400, detail=f"不支持的适配器类型: {adapter_type}")
|
||||
out = {}
|
||||
missing = []
|
||||
for f in schema:
|
||||
k = f["key"]
|
||||
if k in cfg and cfg[k] not in (None, ""):
|
||||
out[k] = cfg[k]
|
||||
elif "default" in f:
|
||||
out[k] = f["default"]
|
||||
elif f.get("required"):
|
||||
missing.append(k)
|
||||
if missing:
|
||||
raise HTTPException(400, detail="缺少必填配置字段: " + ", ".join(missing))
|
||||
if adapter_type in ("alist", "openlist"):
|
||||
username = out.get("username")
|
||||
password = out.get("password")
|
||||
if (username and not password) or (password and not username):
|
||||
raise HTTPException(400, detail="用户名和密码必须同时填写或同时留空")
|
||||
return out
|
||||
|
||||
@classmethod
|
||||
async def create_adapter(cls, data: AdapterCreate, current_user: Optional[User]):
|
||||
norm_path = AdapterCreate.normalize_mount_path(data.path)
|
||||
exists = await StorageAdapter.get_or_none(path=norm_path)
|
||||
if exists:
|
||||
raise HTTPException(400, detail="Mount path already exists")
|
||||
|
||||
adapter_fields = {
|
||||
"name": data.name,
|
||||
"type": data.type,
|
||||
"config": cls._validate_and_normalize_config(data.type, data.config or {}),
|
||||
"enabled": data.enabled,
|
||||
"path": norm_path,
|
||||
"sub_path": data.sub_path,
|
||||
}
|
||||
|
||||
rec = await StorageAdapter.create(**adapter_fields)
|
||||
await runtime_registry.upsert(rec)
|
||||
return AdapterOut.model_validate(rec)
|
||||
|
||||
@classmethod
|
||||
async def list_adapters(cls):
|
||||
adapters = await StorageAdapter.all()
|
||||
return [AdapterOut.model_validate(a) for a in adapters]
|
||||
|
||||
@classmethod
|
||||
async def available_adapter_types(cls):
|
||||
data = []
|
||||
for adapter_type, fields in get_config_schemas().items():
|
||||
data.append({
|
||||
"type": adapter_type,
|
||||
"config_schema": fields,
|
||||
})
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
async def get_adapter(cls, adapter_id: int):
|
||||
rec = await StorageAdapter.get_or_none(id=adapter_id)
|
||||
if not rec:
|
||||
raise HTTPException(404, detail="Not found")
|
||||
return AdapterOut.model_validate(rec)
|
||||
|
||||
@classmethod
|
||||
async def update_adapter(cls, adapter_id: int, data: AdapterCreate, current_user: Optional[User]):
|
||||
rec = await StorageAdapter.get_or_none(id=adapter_id)
|
||||
if not rec:
|
||||
raise HTTPException(404, detail="Not found")
|
||||
|
||||
norm_path = AdapterCreate.normalize_mount_path(data.path)
|
||||
existing = await StorageAdapter.get_or_none(path=norm_path)
|
||||
if existing and existing.id != adapter_id:
|
||||
raise HTTPException(400, detail="Mount path already exists")
|
||||
|
||||
rec.name = data.name
|
||||
rec.type = data.type
|
||||
rec.config = cls._validate_and_normalize_config(data.type, data.config or {})
|
||||
rec.enabled = data.enabled
|
||||
rec.path = norm_path
|
||||
rec.sub_path = data.sub_path
|
||||
await rec.save()
|
||||
|
||||
await runtime_registry.upsert(rec)
|
||||
return AdapterOut.model_validate(rec)
|
||||
|
||||
@classmethod
|
||||
async def delete_adapter(cls, adapter_id: int, current_user: Optional[User]):
|
||||
deleted = await StorageAdapter.filter(id=adapter_id).delete()
|
||||
if not deleted:
|
||||
raise HTTPException(404, detail="Not found")
|
||||
runtime_registry.remove(adapter_id)
|
||||
return {"deleted": True}
|
||||
@@ -0,0 +1,50 @@
|
||||
import re
|
||||
from typing import Dict, Optional
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
class AdapterBase(BaseModel):
|
||||
name: str
|
||||
type: str = Field(pattern=r"^[a-z0-9_]+$")
|
||||
config: Dict = Field(default_factory=dict)
|
||||
enabled: bool = True
|
||||
path: str = None
|
||||
sub_path: Optional[str] = None
|
||||
|
||||
@field_validator("type", mode="before")
|
||||
@classmethod
|
||||
def _normalize_type(cls, v: str):
|
||||
if not isinstance(v, str):
|
||||
raise ValueError("type required")
|
||||
normalized = v.strip().lower()
|
||||
if not normalized:
|
||||
raise ValueError("type required")
|
||||
if not re.fullmatch(r"[a-z0-9_]+", normalized):
|
||||
raise ValueError("type must be lowercase alphanumeric or underscore")
|
||||
return normalized
|
||||
|
||||
|
||||
class AdapterCreate(AdapterBase):
|
||||
@staticmethod
|
||||
def normalize_mount_path(p: str) -> str:
|
||||
p = p.strip()
|
||||
if not p.startswith('/'):
|
||||
p = '/' + p
|
||||
p = p.rstrip('/')
|
||||
return p or '/'
|
||||
|
||||
@field_validator("path")
|
||||
def _v_mount(cls, v: str):
|
||||
if not v:
|
||||
raise ValueError("mount_path required")
|
||||
return cls.normalize_mount_path(v)
|
||||
|
||||
|
||||
class AdapterOut(AdapterBase):
|
||||
id: int
|
||||
path: str = None
|
||||
sub_path: Optional[str] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
Reference in New Issue
Block a user