refactor: optimize backend module

This commit is contained in:
shiyu
2025-12-08 17:46:45 +08:00
parent cf8d10f71c
commit 8f515aaaf4
124 changed files with 6884 additions and 6390 deletions
+3
View File
@@ -0,0 +1,3 @@
from .base import BaseAdapter
__all__ = ["BaseAdapter"]
+24
View File
@@ -0,0 +1,24 @@
from __future__ import annotations
from typing import List, Dict, Protocol, runtime_checkable, Tuple, AsyncIterator
from models import StorageAdapter
# 约定:任意新适配器模块需定义:
# ADAPTER_TYPE: str
# CONFIG_SCHEMA: List[Dict]
# ADAPTER_FACTORY: Callable[[StorageAdapter], BaseAdapter] (可省略, 会自动寻找 *Adapter 类)
@runtime_checkable
class BaseAdapter(Protocol):
record: StorageAdapter
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]: ...
async def read_file(self, root: str, rel: str) -> bytes: ...
async def write_file(self, root: str, rel: str, data: bytes): ...
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]): ...
async def mkdir(self, root: str, rel: str): ...
async def delete(self, root: str, rel: str): ...
async def move(self, root: str, src_rel: str, dst_rel: str): ...
async def rename(self, root: str, src_rel: str, dst_rel: str): ...
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False): ...
async def stream_file(self, root: str, rel: str, range_header: str | None): ...
async def stat_file(self, root: str, rel: str): ...
def get_effective_root(self, sub_path: str | None) -> str: ...
+614
View File
@@ -0,0 +1,614 @@
from __future__ import annotations
import asyncio
from dataclasses import dataclass
from typing import List, Dict, Tuple, AsyncIterator, Optional
from fastapi import HTTPException
from fastapi.responses import StreamingResponse
from ftplib import FTP, error_perm
import mimetypes
from models import StorageAdapter
def _join_remote(root: str, rel: str) -> str:
root = (root or "/").rstrip("/") or "/"
rel = (rel or "").lstrip("/")
if not rel:
return root
return f"{root}/{rel}"
def _parse_mlst_line(line: str) -> Dict[str, str]:
out: Dict[str, str] = {}
try:
facts, _, name = line.partition(" ")
for part in facts.split(";"):
if not part or "=" not in part:
continue
k, v = part.split("=", 1)
out[k.strip().lower()] = v.strip()
if name:
out["name"] = name.strip()
except Exception:
pass
return out
def _parse_modify_to_epoch(mod: str) -> int:
# Formats we may see: YYYYMMDDHHMMSS or YYYYMMDDHHMMSS(.sss)
try:
mod = mod.strip()
mod = mod.split(".")[0]
if len(mod) >= 14:
y = int(mod[0:4])
m = int(mod[4:6])
d = int(mod[6:8])
hh = int(mod[8:10])
mm = int(mod[10:12])
ss = int(mod[12:14])
import datetime as _dt
return int(_dt.datetime(y, m, d, hh, mm, ss, tzinfo=_dt.timezone.utc).timestamp())
except Exception:
return 0
return 0
@dataclass
class _Range:
start: int
end: Optional[int] # inclusive
class FTPAdapter:
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config
self.host: str = cfg.get("host")
self.port: int = int(cfg.get("port", 21))
self.username: Optional[str] = cfg.get("username")
self.password: Optional[str] = cfg.get("password")
self.passive: bool = bool(cfg.get("passive", True))
self.timeout: int = int(cfg.get("timeout", 15))
self.root_path: str = cfg.get("root", "/") or "/"
if not self.host:
raise ValueError("FTP adapter requires 'host'")
def get_effective_root(self, sub_path: str | None) -> str:
base = self.root_path.rstrip("/") or "/"
if sub_path:
return _join_remote(base, sub_path)
return base
def _connect(self) -> FTP:
ftp = FTP()
ftp.connect(self.host, self.port, timeout=self.timeout)
if self.username:
ftp.login(self.username, self.password or "")
else:
ftp.login()
ftp.set_pasv(self.passive)
return ftp
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
path = _join_remote(root, rel.strip('/'))
def _do_list() -> List[Dict]:
ftp = self._connect()
try:
ftp.cwd(path)
except error_perm as e:
# path may be file
ftp.quit()
raise NotADirectoryError(rel) from e
entries: List[Dict] = []
# Try MLSD first
try:
for name, facts in ftp.mlsd():
if name in (".", ".."):
continue
is_dir = (facts.get("type") == "dir")
size = int(facts.get("size") or 0)
mtime = _parse_modify_to_epoch(facts.get("modify") or "")
entries.append({
"name": name,
"is_dir": is_dir,
"size": 0 if is_dir else size,
"mtime": mtime,
"type": "dir" if is_dir else "file",
})
ftp.quit()
return entries
except Exception:
# Fallback to NLST + probing
pass
names = []
try:
names = ftp.nlst()
except Exception:
ftp.quit()
return []
for name in names:
if name in (".", ".."):
continue
is_dir = False
size = 0
mtime = 0
try:
# If we can CWD, it's a directory
ftp.cwd(_join_remote(path, name))
ftp.cwd(path)
is_dir = True
except Exception:
is_dir = False
try:
size = ftp.size(_join_remote(path, name)) or 0
except Exception:
size = 0
try:
mdtm = ftp.sendcmd("MDTM " + _join_remote(path, name))
# Example: '213 20241012XXXXXX'
if mdtm.startswith("213 "):
mtime = _parse_modify_to_epoch(mdtm.split(" ", 1)[1])
except Exception:
pass
entries.append({
"name": name,
"is_dir": is_dir,
"size": 0 if is_dir else int(size or 0),
"mtime": int(mtime or 0),
"type": "dir" if is_dir else "file",
})
ftp.quit()
return entries
entries = await asyncio.to_thread(_do_list)
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
key = (not item["is_dir"],)
f = sort_by.lower()
if f == "name":
key += (item["name"].lower(),)
elif f == "size":
key += (item.get("size", 0),)
elif f == "mtime":
key += (item.get("mtime", 0),)
else:
key += (item["name"].lower(),)
return key
entries.sort(key=get_sort_key, reverse=reverse)
total = len(entries)
start = (page_num - 1) * page_size
end = start + page_size
return entries[start:end], total
async def read_file(self, root: str, rel: str) -> bytes:
path = _join_remote(root, rel)
def _do_read() -> bytes:
ftp = self._connect()
try:
chunks: List[bytes] = []
ftp.retrbinary("RETR " + path, lambda b: chunks.append(b))
return b"".join(chunks)
except error_perm as e:
if str(e).startswith("550"):
raise FileNotFoundError(rel)
raise
finally:
try:
ftp.quit()
except Exception:
pass
return await asyncio.to_thread(_do_read)
async def write_file(self, root: str, rel: str, data: bytes):
path = _join_remote(root, rel)
def _ensure_dirs(ftp: FTP, dir_path: str):
parts = [p for p in dir_path.strip("/").split("/") if p]
cur = "/"
for p in parts:
cur = _join_remote(cur, p)
try:
ftp.mkd(cur)
except Exception:
pass
def _do_write():
ftp = self._connect()
try:
parent = "/" if "/" not in path.strip("/") else path.rsplit("/", 1)[0]
_ensure_dirs(ftp, parent)
from io import BytesIO
bio = BytesIO(data)
ftp.storbinary("STOR " + path, bio)
finally:
try:
ftp.quit()
except Exception:
pass
await asyncio.to_thread(_do_write)
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
# KISS: 聚合后一次性写入
buf = bytearray()
async for chunk in data_iter:
if chunk:
buf.extend(chunk)
await self.write_file(root, rel, bytes(buf))
return len(buf)
async def mkdir(self, root: str, rel: str):
path = _join_remote(root, rel)
def _do_mkdir():
ftp = self._connect()
try:
parts = [p for p in path.strip("/").split("/") if p]
cur = "/"
for p in parts:
cur = _join_remote(cur, p)
try:
ftp.mkd(cur)
except Exception:
pass
finally:
try:
ftp.quit()
except Exception:
pass
await asyncio.to_thread(_do_mkdir)
async def delete(self, root: str, rel: str):
path = _join_remote(root, rel)
def _do_delete():
ftp = self._connect()
try:
# Try file delete
try:
ftp.delete(path)
return
except Exception:
pass
# Recursively delete dir
def _rm_tree(dir_path: str):
try:
ftp.cwd(dir_path)
except Exception:
return
items = []
try:
for name, facts in ftp.mlsd():
if name in (".", ".."):
continue
items.append((name, facts.get("type") == "dir"))
except Exception:
try:
names = ftp.nlst()
except Exception:
names = []
for n in names:
if n in (".", ".."):
continue
# Best-effort dir check
try:
ftp.cwd(_join_remote(dir_path, n))
ftp.cwd(dir_path)
items.append((n, True))
except Exception:
items.append((n, False))
for n, is_dir in items:
child = _join_remote(dir_path, n)
if is_dir:
_rm_tree(child)
else:
try:
ftp.delete(child)
except Exception:
pass
try:
ftp.rmd(dir_path)
except Exception:
pass
_rm_tree(path)
finally:
try:
ftp.quit()
except Exception:
pass
await asyncio.to_thread(_do_delete)
async def move(self, root: str, src_rel: str, dst_rel: str):
src = _join_remote(root, src_rel)
dst = _join_remote(root, dst_rel)
def _do_move():
ftp = self._connect()
try:
# Ensure dst parent exists
parent = "/" if "/" not in dst.strip("/") else dst.rsplit("/", 1)[0]
parts = [p for p in parent.strip("/").split("/") if p]
cur = "/"
for p in parts:
cur = _join_remote(cur, p)
try:
ftp.mkd(cur)
except Exception:
pass
ftp.rename(src, dst)
finally:
try:
ftp.quit()
except Exception:
pass
await asyncio.to_thread(_do_move)
async def rename(self, root: str, src_rel: str, dst_rel: str):
await self.move(root, src_rel, dst_rel)
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
src = _join_remote(root, src_rel)
dst = _join_remote(root, dst_rel)
# naive implementation: download then upload; recursively for dirs
async def _is_dir(path: str) -> bool:
def _probe() -> bool:
ftp = self._connect()
try:
try:
ftp.cwd(path)
return True
except Exception:
return False
finally:
try:
ftp.quit()
except Exception:
pass
return await asyncio.to_thread(_probe)
if await _is_dir(src):
# list children, create dst dir, copy recursively
await self.mkdir(root, dst_rel)
children, _ = await self.list_dir(root, src_rel, page_num=1, page_size=10_000)
for ent in children:
child_src = f"{src_rel.rstrip('/')}/{ent['name']}"
child_dst = f"{dst_rel.rstrip('/')}/{ent['name']}"
await self.copy(root, child_src, child_dst, overwrite)
return
# file
data = await self.read_file(root, src_rel)
if not overwrite:
# best-effort existence check
try:
await self.stat_file(root, dst_rel)
raise FileExistsError(dst_rel)
except FileNotFoundError:
pass
await self.write_file(root, dst_rel, data)
async def stat_file(self, root: str, rel: str):
path = _join_remote(root, rel)
def _do_stat():
ftp = self._connect()
try:
# Try MLST
try:
resp: List[str] = []
ftp.retrlines("MLST " + path, resp.append)
# The last line usually contains facts
facts = {}
if resp:
facts = _parse_mlst_line(resp[-1])
name = rel.split("/")[-1]
t = facts.get("type") or "file"
is_dir = t == "dir"
size = int(facts.get("size") or 0)
mtime = _parse_modify_to_epoch(facts.get("modify") or "")
return {
"name": name,
"is_dir": is_dir,
"size": 0 if is_dir else size,
"mtime": mtime,
"type": "dir" if is_dir else "file",
"path": path,
}
except Exception:
pass
# Probe directory
try:
ftp.cwd(path)
return {
"name": rel.split("/")[-1],
"is_dir": True,
"size": 0,
"mtime": 0,
"type": "dir",
"path": path,
}
except Exception:
pass
# Treat as file
try:
size = ftp.size(path) or 0
except Exception:
size = 0
try:
mdtm = ftp.sendcmd("MDTM " + path)
mtime = _parse_modify_to_epoch(mdtm.split(" ", 1)[1]) if mdtm.startswith("213 ") else 0
except Exception:
mtime = 0
return {
"name": rel.split("/")[-1],
"is_dir": False,
"size": int(size or 0),
"mtime": int(mtime or 0),
"type": "file",
"path": path,
}
except error_perm as e:
if str(e).startswith("550"):
raise FileNotFoundError(rel)
raise
finally:
try:
ftp.quit()
except Exception:
pass
return await asyncio.to_thread(_do_stat)
async def stream_file(self, root: str, rel: str, range_header: str | None):
path = _join_remote(root, rel)
# Get size (best-effort)
def _get_size() -> Optional[int]:
ftp = self._connect()
try:
try:
return int(ftp.size(path) or 0)
except Exception:
return None
finally:
try:
ftp.quit()
except Exception:
pass
total_size = await asyncio.to_thread(_get_size)
mime, _ = mimetypes.guess_type(rel)
content_type = mime or "application/octet-stream"
rng: Optional[_Range] = None
status = 200
headers = {"Accept-Ranges": "bytes", "Content-Type": content_type}
if range_header and range_header.startswith("bytes=") and total_size is not None:
try:
s, e = (range_header.removeprefix("bytes=").split("-", 1))
start = int(s) if s.strip() else 0
end = int(e) if e.strip() else (total_size - 1)
if start >= total_size:
raise HTTPException(416, detail="Requested Range Not Satisfiable")
if end >= total_size:
end = total_size - 1
rng = _Range(start, end)
status = 206
headers["Content-Range"] = f"bytes {start}-{end}/{total_size}"
headers["Content-Length"] = str(end - start + 1)
except ValueError:
raise HTTPException(400, detail="Invalid Range header")
elif total_size is not None:
headers["Content-Length"] = str(total_size)
queue: asyncio.Queue[Optional[bytes]] = asyncio.Queue(maxsize=8)
class _Stop(Exception):
pass
def _worker():
ftp = self._connect()
remaining = None
if rng is not None:
remaining = (rng.end - rng.start + 1) if rng.end is not None else None
def _cb(chunk: bytes):
nonlocal remaining
if not chunk:
return
try:
if remaining is not None:
if len(chunk) > remaining:
part = chunk[:remaining]
queue.put_nowait(part)
remaining = 0
raise _Stop()
else:
queue.put_nowait(chunk)
remaining -= len(chunk)
if remaining <= 0:
raise _Stop()
else:
queue.put_nowait(chunk)
except _Stop:
raise
except Exception:
# queue full or event loop closed
raise _Stop()
try:
if rng is not None:
ftp.retrbinary("RETR " + path, _cb, rest=rng.start)
else:
ftp.retrbinary("RETR " + path, _cb)
queue.put_nowait(None)
except _Stop:
try:
queue.put_nowait(None)
except Exception:
pass
except error_perm as e:
try:
queue.put_nowait(None)
except Exception:
pass
if str(e).startswith("550"):
pass
finally:
try:
ftp.quit()
except Exception:
pass
async def agen():
worker_fut = asyncio.to_thread(_worker)
try:
while True:
chunk = await queue.get()
if chunk is None:
break
yield chunk
finally:
try:
await worker_fut
except Exception:
pass
return StreamingResponse(agen(), status_code=status, headers=headers, media_type=content_type)
ADAPTER_TYPE = "ftp"
CONFIG_SCHEMA = [
{"key": "host", "label": "主机", "type": "string", "required": True, "placeholder": "ftp.example.com"},
{"key": "port", "label": "端口", "type": "number", "required": False, "default": 21},
{"key": "username", "label": "用户名", "type": "string", "required": False},
{"key": "password", "label": "密码", "type": "password", "required": False},
{"key": "passive", "label": "被动模式", "type": "boolean", "required": False, "default": True},
{"key": "timeout", "label": "超时(秒)", "type": "number", "required": False, "default": 15},
{"key": "root", "label": "根路径", "type": "string", "required": False, "default": "/"},
]
def ADAPTER_FACTORY(rec: StorageAdapter):
return FTPAdapter(rec)
+560
View File
@@ -0,0 +1,560 @@
from __future__ import annotations
from datetime import datetime, timezone, timedelta
from typing import List, Dict, Tuple, AsyncIterator
import httpx
from fastapi.responses import StreamingResponse, Response
from fastapi import HTTPException
from models import StorageAdapter
GOOGLE_OAUTH_URL = "https://oauth2.googleapis.com/token"
GOOGLE_DRIVE_API_URL = "https://www.googleapis.com/drive/v3"
class GoogleDriveAdapter:
"""Google Drive 存储适配器"""
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config
self.client_id = cfg.get("client_id")
self.client_secret = cfg.get("client_secret")
self.refresh_token = cfg.get("refresh_token")
self.root_folder_id = cfg.get("root_folder_id", "root")
self.enable_redirect_307 = bool(cfg.get("enable_direct_download_307"))
if not all([self.client_id, self.client_secret, self.refresh_token]):
raise ValueError(
"Google Drive 适配器需要 client_id, client_secret, 和 refresh_token")
self._access_token: str | None = None
self._token_expiry: datetime | None = None
def get_effective_root(self, sub_path: str | None) -> str:
"""
获取有效根路径。
:param sub_path: 子路径。
:return: 完整的有效路径。
"""
if sub_path:
return f"{sub_path.strip('/')}".strip()
return ""
async def _get_access_token(self) -> str:
"""
获取或刷新 access token。
:return: access token。
"""
if self._access_token and self._token_expiry and datetime.now(timezone.utc) < self._token_expiry:
return self._access_token
data = {
"client_id": self.client_id,
"client_secret": self.client_secret,
"refresh_token": self.refresh_token,
"grant_type": "refresh_token",
}
async with httpx.AsyncClient(timeout=20.0) as client:
resp = await client.post(GOOGLE_OAUTH_URL, data=data)
resp.raise_for_status()
token_data = resp.json()
self._access_token = token_data["access_token"]
self._token_expiry = datetime.now(
timezone.utc) + timedelta(seconds=token_data["expires_in"] - 300)
return self._access_token
async def _request(self, method: str, endpoint: str, **kwargs):
"""
向 Google Drive API 发送请求。
:param method: HTTP 方法。
:param endpoint: API 端点。
:param kwargs: 其他请求参数。
:return: 响应对象。
"""
token = await self._get_access_token()
headers = {"Authorization": f"Bearer {token}"}
if "headers" in kwargs:
headers.update(kwargs.pop("headers"))
url = f"{GOOGLE_DRIVE_API_URL}{endpoint}"
async with httpx.AsyncClient(timeout=60.0) as client:
resp = await client.request(method, url, headers=headers, **kwargs)
if resp.status_code == 401:
self._access_token = None
token = await self._get_access_token()
headers["Authorization"] = f"Bearer {token}"
resp = await client.request(method, url, headers=headers, **kwargs)
return resp
async def _get_folder_id_by_path(self, path: str) -> str:
"""
通过路径获取文件夹 ID。
:param path: 路径。
:return: 文件夹 ID。
"""
if not path or path == "/":
return self.root_folder_id
parts = [p for p in path.strip("/").split("/") if p]
current_id = self.root_folder_id
for part in parts:
query = f"name='{part}' and '{current_id}' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false"
params = {"q": query, "fields": "files(id, name)"}
resp = await self._request("GET", "/files", params=params)
resp.raise_for_status()
data = resp.json()
files = data.get("files", [])
if not files:
raise FileNotFoundError(f"文件夹不存在: {part}")
current_id = files[0]["id"]
return current_id
async def _get_file_id_by_path(self, path: str) -> str | None:
"""
通过路径获取文件 ID。
:param path: 文件路径。
:return: 文件 ID 或 None。
"""
if not path or path == "/":
return self.root_folder_id
parts = [p for p in path.strip("/").split("/") if p]
parent_id = self.root_folder_id
for i, part in enumerate(parts):
is_last = i == len(parts) - 1
mime_filter = "" if is_last else "and mimeType='application/vnd.google-apps.folder'"
query = f"name='{part}' and '{parent_id}' in parents {mime_filter} and trashed=false"
params = {"q": query, "fields": "files(id, name)"}
resp = await self._request("GET", "/files", params=params)
resp.raise_for_status()
data = resp.json()
files = data.get("files", [])
if not files:
return None
parent_id = files[0]["id"]
return parent_id
def _format_item(self, item: Dict) -> Dict:
"""
将 Google Drive API 返回的 item 格式化为统一的格式。
:param item: Google Drive API 返回的 item 字典。
:return: 格式化后的字典。
"""
is_dir = item["mimeType"] == "application/vnd.google-apps.folder"
mtime_str = item.get("modifiedTime", item.get("createdTime", ""))
try:
mtime = int(datetime.fromisoformat(mtime_str.replace("Z", "+00:00")).timestamp())
except:
mtime = 0
return {
"name": item["name"],
"is_dir": is_dir,
"size": 0 if is_dir else int(item.get("size", 0)),
"mtime": mtime,
"type": "dir" if is_dir else "file",
}
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
"""
列出目录内容。
:param root: 根路径。
:param rel: 相对路径。
:param page_num: 页码。
:param page_size: 每页大小。
:param sort_by: 排序字段
:param sort_order: 排序顺序
:return: 文件/目录列表和总数。
"""
try:
folder_id = await self._get_folder_id_by_path(rel)
except FileNotFoundError:
return [], 0
query = f"'{folder_id}' in parents and trashed=false"
params = {
"q": query,
"fields": "files(id, name, mimeType, size, modifiedTime, createdTime)",
"pageSize": 1000,
}
all_items = []
page_token = None
while True:
if page_token:
params["pageToken"] = page_token
resp = await self._request("GET", "/files", params=params)
if resp.status_code == 404:
return [], 0
resp.raise_for_status()
data = resp.json()
all_items.extend(data.get("files", []))
page_token = data.get("nextPageToken")
if not page_token:
break
formatted_items = [self._format_item(item) for item in all_items]
# 排序
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
key = (not item["is_dir"],)
sort_field = sort_by.lower()
if sort_field == "name":
key += (item["name"].lower(),)
elif sort_field == "size":
key += (item["size"],)
elif sort_field == "mtime":
key += (item["mtime"],)
else:
key += (item["name"].lower(),)
return key
formatted_items.sort(key=get_sort_key, reverse=reverse)
total_count = len(formatted_items)
start_idx = (page_num - 1) * page_size
end_idx = start_idx + page_size
return formatted_items[start_idx:end_idx], total_count
async def read_file(self, root: str, rel: str) -> bytes:
"""
读取文件内容。
:param root: 根路径。
:param rel: 相对路径。
:return: 文件内容的字节流。
"""
file_id = await self._get_file_id_by_path(rel)
if not file_id:
raise FileNotFoundError(rel)
resp = await self._request("GET", f"/files/{file_id}", params={"alt": "media"})
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
return resp.content
async def write_file(self, root: str, rel: str, data: bytes):
"""
写入文件。
:param root: 根路径。
:param rel: 相对路径。
:param data: 文件内容的字节流。
"""
parent_path = "/".join(rel.strip("/").split("/")[:-1])
file_name = rel.strip("/").split("/")[-1]
parent_id = await self._get_folder_id_by_path(parent_path)
# 检查文件是否已存在
existing_id = await self._get_file_id_by_path(rel)
if existing_id:
# 更新现有文件
async with httpx.AsyncClient(timeout=60.0) as client:
token = await self._get_access_token()
headers = {"Authorization": f"Bearer {token}"}
url = f"https://www.googleapis.com/upload/drive/v3/files/{existing_id}?uploadType=media"
resp = await client.patch(url, headers=headers, content=data)
resp.raise_for_status()
else:
# 创建新文件
metadata = {
"name": file_name,
"parents": [parent_id]
}
async with httpx.AsyncClient(timeout=60.0) as client:
token = await self._get_access_token()
headers = {"Authorization": f"Bearer {token}"}
# 使用 multipart 上传
import json
boundary = "===============boundary==============="
headers["Content-Type"] = f"multipart/related; boundary={boundary}"
body = (
f"--{boundary}\r\n"
f"Content-Type: application/json; charset=UTF-8\r\n\r\n"
f"{json.dumps(metadata)}\r\n"
f"--{boundary}\r\n"
f"Content-Type: application/octet-stream\r\n\r\n"
).encode() + data + f"\r\n--{boundary}--".encode()
url = "https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart"
resp = await client.post(url, headers=headers, content=body)
resp.raise_for_status()
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
"""
以流式方式写入文件。
:param root: 根路径。
:param rel: 相对路径。
:param data_iter: 文件内容的异步迭代器。
:return: 文件大小。
"""
# 先收集所有数据
chunks = []
total_size = 0
async for chunk in data_iter:
chunks.append(chunk)
total_size += len(chunk)
data = b"".join(chunks)
await self.write_file(root, rel, data)
return total_size
async def mkdir(self, root: str, rel: str):
"""
创建目录。
:param root: 根路径。
:param rel: 相对路径。
"""
parent_path = "/".join(rel.strip("/").split("/")[:-1])
folder_name = rel.strip("/").split("/")[-1]
parent_id = await self._get_folder_id_by_path(parent_path)
metadata = {
"name": folder_name,
"mimeType": "application/vnd.google-apps.folder",
"parents": [parent_id]
}
resp = await self._request("POST", "/files", json=metadata)
resp.raise_for_status()
async def delete(self, root: str, rel: str):
"""
删除文件或目录。
:param root: 根路径。
:param rel: 相对路径。
"""
file_id = await self._get_file_id_by_path(rel)
if not file_id:
return
resp = await self._request("DELETE", f"/files/{file_id}")
if resp.status_code not in (204, 404):
resp.raise_for_status()
async def move(self, root: str, src_rel: str, dst_rel: str):
"""
移动或重命名文件/目录。
:param root: 根路径。
:param src_rel: 源相对路径。
:param dst_rel: 目标相对路径。
"""
file_id = await self._get_file_id_by_path(src_rel)
if not file_id:
raise FileNotFoundError(src_rel)
# 获取当前父文件夹
resp = await self._request("GET", f"/files/{file_id}", params={"fields": "parents"})
resp.raise_for_status()
current_parents = resp.json().get("parents", [])
# 获取目标父文件夹和新名称
dst_parent_path = "/".join(dst_rel.strip("/").split("/")[:-1])
dst_name = dst_rel.strip("/").split("/")[-1]
dst_parent_id = await self._get_folder_id_by_path(dst_parent_path)
# 更新文件
params = {
"addParents": dst_parent_id,
"removeParents": ",".join(current_parents) if current_parents else None,
}
metadata = {"name": dst_name}
resp = await self._request("PATCH", f"/files/{file_id}", params=params, json=metadata)
resp.raise_for_status()
async def rename(self, root: str, src_rel: str, dst_rel: str):
"""
重命名文件或目录。
"""
await self.move(root, src_rel, dst_rel)
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
"""
复制文件或目录。
:param root: 根路径。
:param src_rel: 源相对路径。
:param dst_rel: 目标相对路径。
:param overwrite: 是否覆盖。
"""
file_id = await self._get_file_id_by_path(src_rel)
if not file_id:
raise FileNotFoundError(src_rel)
dst_parent_path = "/".join(dst_rel.strip("/").split("/")[:-1])
dst_name = dst_rel.strip("/").split("/")[-1]
dst_parent_id = await self._get_folder_id_by_path(dst_parent_path)
metadata = {
"name": dst_name,
"parents": [dst_parent_id]
}
resp = await self._request("POST", f"/files/{file_id}/copy", json=metadata)
resp.raise_for_status()
async def stream_file(self, root: str, rel: str, range_header: str | None):
"""
流式传输文件(支持范围请求)。
:param root: 根路径。
:param rel: 相对路径。
:param range_header: HTTP Range 头。
:return: FastAPI StreamingResponse 对象。
"""
file_id = await self._get_file_id_by_path(rel)
if not file_id:
raise FileNotFoundError(rel)
# 获取文件元数据
resp = await self._request("GET", f"/files/{file_id}", params={"fields": "name, size, mimeType"})
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
item_data = resp.json()
file_size = int(item_data.get("size", 0))
content_type = item_data.get("mimeType", "application/octet-stream")
start = 0
end = file_size - 1
status = 200
headers = {
"Accept-Ranges": "bytes",
"Content-Type": content_type,
"Content-Disposition": f"inline; filename=\"{item_data.get('name')}\""
}
if range_header and range_header.startswith("bytes="):
try:
part = range_header.removeprefix("bytes=")
s, e = part.split("-", 1)
if s.strip():
start = int(s)
if e.strip():
end = int(e)
if start >= file_size:
raise HTTPException(416, "Requested Range Not Satisfiable")
if end >= file_size:
end = file_size - 1
status = 206
except ValueError:
raise HTTPException(400, "Invalid Range header")
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
headers["Content-Length"] = str(end - start + 1)
else:
headers["Content-Length"] = str(file_size)
async def file_iterator():
nonlocal start, end
token = await self._get_access_token()
async with httpx.AsyncClient(timeout=60.0) as client:
req_headers = {
'Authorization': f'Bearer {token}',
'Range': f'bytes={start}-{end}'
}
url = f"{GOOGLE_DRIVE_API_URL}/files/{file_id}?alt=media"
async with client.stream("GET", url, headers=req_headers) as stream_resp:
stream_resp.raise_for_status()
async for chunk in stream_resp.aiter_bytes():
yield chunk
return StreamingResponse(file_iterator(), status_code=status, headers=headers, media_type=content_type)
async def stat_file(self, root: str, rel: str):
"""
获取文件或目录的元数据。
:param root: 根路径。
:param rel: 相对路径。
:return: 格式化后的文件/目录信息。
"""
file_id = await self._get_file_id_by_path(rel)
if not file_id:
raise FileNotFoundError(rel)
resp = await self._request("GET", f"/files/{file_id}", params={"fields": "id, name, mimeType, size, modifiedTime, createdTime"})
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
return self._format_item(resp.json())
async def get_direct_download_response(self, root: str, rel: str):
"""
获取直接下载响应 (307 重定向)。
:param root: 根路径。
:param rel: 相对路径。
:return: 307 重定向响应或 None。
"""
if not self.enable_redirect_307:
return None
file_id = await self._get_file_id_by_path(rel)
if not file_id:
raise FileNotFoundError(rel)
# 获取文件的下载链接
resp = await self._request("GET", f"/files/{file_id}", params={"fields": "webContentLink"})
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
item_data = resp.json()
download_url = item_data.get("webContentLink")
if not download_url:
return None
return Response(status_code=307, headers={"Location": download_url})
async def get_thumbnail(self, root: str, rel: str, size: str = "medium"):
"""
获取文件的缩略图。
:param root: 根路径。
:param rel: 相对路径。
:param size: 缩略图大小 (暂未使用,Google Drive 自动决定)。
:return: 缩略图内容的字节流,或在不支持时返回 None。
"""
file_id = await self._get_file_id_by_path(rel)
if not file_id:
return None
try:
resp = await self._request("GET", f"/files/{file_id}", params={"fields": "thumbnailLink"})
if resp.status_code == 200:
item_data = resp.json()
thumbnail_link = item_data.get("thumbnailLink")
if thumbnail_link:
async with httpx.AsyncClient(timeout=30.0) as client:
thumb_resp = await client.get(thumbnail_link)
thumb_resp.raise_for_status()
return thumb_resp.content
return None
except Exception:
return None
ADAPTER_TYPE = "googledrive"
CONFIG_SCHEMA = [
{"key": "client_id", "label": "Client ID", "type": "string", "required": True},
{"key": "client_secret", "label": "Client Secret",
"type": "password", "required": True},
{"key": "refresh_token", "label": "Refresh Token", "type": "password",
"required": True, "help_text": "可以通过 Google OAuth 2.0 Playground 获取"},
{"key": "root_folder_id", "label": "根文件夹 ID (Root Folder ID)", "type": "string",
"required": False, "placeholder": "默认为根目录 (root)", "default": "root"},
{"key": "enable_direct_download_307", "label": "Enable 307 redirect download", "type": "boolean", "default": False},
]
def ADAPTER_FACTORY(rec): return GoogleDriveAdapter(rec)
+312
View File
@@ -0,0 +1,312 @@
from __future__ import annotations
import os
import shutil
import stat
import time
from pathlib import Path
from typing import List, Dict, Tuple, AsyncIterator
import asyncio
import mimetypes
from fastapi import HTTPException
from fastapi.responses import StreamingResponse, Response
from models import StorageAdapter
def _safe_join(root: str, rel: str) -> Path:
root_path = Path(root).resolve()
full = (root_path / rel).resolve()
if not str(full).startswith(str(root_path)):
raise ValueError("Path escape detected")
return full
DEFAULT_FILE_MODE = 0o666
DEFAULT_DIR_MODE = 0o777
def _apply_mode(path: Path, mode: int):
try:
os.chmod(path, mode)
except Exception:
pass
class LocalAdapter:
def __init__(self, record: StorageAdapter):
self.record = record
self.root = self.record.config.get("root")
if not self.root:
raise ValueError("Local adapter config requires 'root'")
Path(self.root).mkdir(parents=True, exist_ok=True)
def get_effective_root(self, sub_path: str | None) -> str:
root = self.record.config.get("root")
if sub_path:
return str(Path(root) / sub_path)
return root
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
rel = rel.strip('/')
base = _safe_join(root, rel) if rel else Path(root)
if not base.exists():
return [], 0
if not base.is_dir():
raise NotADirectoryError(rel)
all_names = await asyncio.to_thread(os.listdir, base)
entries = []
for name in all_names:
fp = base / name
try:
st = await asyncio.to_thread(fp.stat)
except FileNotFoundError:
continue
is_dir = fp.is_dir()
entries.append({
"name": name,
"is_dir": is_dir,
"size": 0 if is_dir else st.st_size,
"mtime": int(st.st_mtime),
"mode": stat.S_IMODE(st.st_mode),
"type": "dir" if is_dir else "file",
})
# 排序
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
# 基础排序键,目录优先
key = (not item["is_dir"],)
sort_field = sort_by.lower()
if sort_field == "name":
key += (item["name"].lower(),)
elif sort_field == "size":
key += (item["size"],)
elif sort_field == "mtime":
key += (item["mtime"],)
else: # 默认按名称
key += (item["name"].lower(),)
return key
entries.sort(key=get_sort_key, reverse=reverse)
total_count = len(entries)
# 分页
start_idx = (page_num - 1) * page_size
end_idx = start_idx + page_size
page_entries = entries[start_idx:end_idx]
return page_entries, total_count
async def read_file(self, root: str, rel: str) -> bytes:
fp = _safe_join(root, rel)
if not fp.exists() or not fp.is_file():
raise FileNotFoundError(rel)
return await asyncio.to_thread(fp.read_bytes)
async def write_file(self, root: str, rel: str, data: bytes):
fp = _safe_join(root, rel)
pre_exists = fp.exists()
await asyncio.to_thread(os.makedirs, fp.parent, mode=DEFAULT_DIR_MODE, exist_ok=True)
await asyncio.to_thread(fp.write_bytes, data)
if not pre_exists:
await asyncio.to_thread(_apply_mode, fp, DEFAULT_FILE_MODE)
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
fp = _safe_join(root, rel)
pre_exists = fp.exists()
await asyncio.to_thread(os.makedirs, fp.parent, mode=DEFAULT_DIR_MODE, exist_ok=True)
# 流式写入,避免一次性读入内存
def _open():
return open(fp, "wb")
f = await asyncio.to_thread(_open)
size = 0
try:
async for chunk in data_iter:
if not chunk:
continue
size += len(chunk)
await asyncio.to_thread(f.write, chunk)
finally:
await asyncio.to_thread(f.close)
if not pre_exists:
await asyncio.to_thread(_apply_mode, fp, DEFAULT_FILE_MODE)
return size
async def mkdir(self, root: str, rel: str):
fp = _safe_join(root, rel)
await asyncio.to_thread(os.makedirs, fp, mode=DEFAULT_DIR_MODE, exist_ok=True)
async def delete(self, root: str, rel: str):
fp = _safe_join(root, rel)
if not fp.exists():
return
if fp.is_dir():
await asyncio.to_thread(shutil.rmtree, fp)
else:
await asyncio.to_thread(fp.unlink)
async def stat_path(self, root: str, rel: str):
"""新增: 返回路径状态调试信息"""
fp = _safe_join(root, rel)
def _stat():
if not fp.exists():
return {"exists": False, "is_dir": None, "path": str(fp)}
return {
"exists": True,
"is_dir": fp.is_dir(),
"path": str(fp)
}
return await asyncio.to_thread(_stat)
async def exists(self, root: str, rel: str) -> bool:
"""新增: 判断路径是否存在"""
fp = _safe_join(root, rel)
return await asyncio.to_thread(fp.exists)
async def move(self, root: str, src_rel: str, dst_rel: str):
src = _safe_join(root, src_rel)
dst = _safe_join(root, dst_rel)
if str(src) == str(dst):
return
if not src.exists():
raise FileNotFoundError(src_rel)
await asyncio.to_thread(dst.parent.mkdir, parents=True, exist_ok=True)
def _do_move():
try:
os.replace(src, dst)
except OSError:
shutil.move(str(src), str(dst))
await asyncio.to_thread(_do_move)
async def rename(self, root: str, src_rel: str, dst_rel: str):
src = _safe_join(root, src_rel)
dst = _safe_join(root, dst_rel)
if str(src) == str(dst):
return
if not src.exists():
raise FileNotFoundError(src_rel)
await asyncio.to_thread(dst.parent.mkdir, parents=True, exist_ok=True)
def _do_rename():
try:
os.rename(src, dst)
except OSError:
os.replace(src, dst)
await asyncio.to_thread(_do_rename)
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
src = _safe_join(root, src_rel)
dst = _safe_join(root, dst_rel)
if not src.exists():
raise FileNotFoundError(src_rel)
if str(src) == str(dst):
return
await asyncio.to_thread(dst.parent.mkdir, parents=True, exist_ok=True)
def _do():
if dst.exists():
if not overwrite:
raise FileExistsError(dst_rel)
if dst.is_dir():
shutil.rmtree(dst)
else:
dst.unlink()
if src.is_dir():
shutil.copytree(src, dst)
else:
shutil.copy2(src, dst)
await asyncio.to_thread(_do)
async def stream_file(self, root: str, rel: str, range_header: str | None):
fp = _safe_join(root, rel)
if not fp.exists() or not fp.is_file():
raise HTTPException(404, detail="File not found")
mime, _ = mimetypes.guess_type(rel)
content_type = mime or "application/octet-stream"
file_size = (await asyncio.to_thread(fp.stat)).st_size
start = 0
end = file_size - 1
status = 200
headers = {
"Accept-Ranges": "bytes",
"Content-Type": content_type,
}
if range_header and range_header.startswith("bytes="):
try:
part = range_header.removeprefix("bytes=")
s, e = part.split("-", 1)
if s.strip():
start = int(s)
if e.strip():
end = int(e)
if start >= file_size:
raise HTTPException(416, detail="Requested Range Not Satisfiable")
if end >= file_size:
end = file_size - 1
status = 206
except ValueError:
raise HTTPException(400, detail="Invalid Range header")
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
headers["Content-Length"] = str(end - start + 1)
else:
headers["Content-Length"] = str(file_size)
async def iterator():
# 使用线程池避免阻塞
def _read_segment(offset: int, length: int):
with open(fp, "rb") as f:
f.seek(offset)
return f.read(length)
chunk_size = 256 * 1024
remaining = end - start + 1
offset = start
while remaining > 0:
size = min(chunk_size, remaining)
data = await asyncio.to_thread(_read_segment, offset, size)
if not data:
break
yield data
remaining -= len(data)
offset += len(data)
return StreamingResponse(iterator(), status_code=status, headers=headers, media_type=content_type)
async def stat_file(self, root: str, rel: str):
fp = _safe_join(root, rel)
if not fp.exists():
raise FileNotFoundError(rel)
st = await asyncio.to_thread(fp.stat)
info = {
"name": fp.name,
"is_dir": fp.is_dir(),
"size": st.st_size,
"mtime": int(st.st_mtime),
"mode": stat.S_IMODE(st.st_mode),
"type": "dir" if fp.is_dir() else "file",
"path": str(fp),
}
# exif信息
exif = None
if not fp.is_dir():
mime, _ = mimetypes.guess_type(fp.name)
if mime and mime.startswith("image/"):
try:
from PIL import Image
img = await asyncio.to_thread(Image.open, fp)
exif_data = img._getexif()
if exif_data:
exif = {str(k): str(v) for k, v in exif_data.items()}
except Exception:
exif = None
info["exif"] = exif
return info
ADAPTER_TYPE = "local"
CONFIG_SCHEMA = [
{"key": "root", "label": "根目录", "type": "string", "required": True, "placeholder": "/data/storage"},
]
ADAPTER_FACTORY = lambda rec: LocalAdapter(rec)
+462
View File
@@ -0,0 +1,462 @@
from __future__ import annotations
from datetime import datetime, timezone, timedelta
from typing import List, Dict, Tuple, AsyncIterator
import httpx
from fastapi.responses import StreamingResponse, Response
from fastapi import HTTPException
from models import StorageAdapter
MS_GRAPH_URL = "https://graph.microsoft.com/v1.0"
MS_OAUTH_URL = "https://login.microsoftonline.com/common/oauth2/v2.0/token"
class OneDriveAdapter:
"""OneDrive 存储适配器"""
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config
self.client_id = cfg.get("client_id")
self.client_secret = cfg.get("client_secret")
self.refresh_token = cfg.get("refresh_token")
self.root = cfg.get("root", "/").strip("/")
self.enable_redirect_307 = bool(cfg.get("enable_direct_download_307"))
if not all([self.client_id, self.client_secret, self.refresh_token]):
raise ValueError(
"OneDrive 适配器需要 client_id, client_secret, 和 refresh_token")
self._access_token: str | None = None
self._token_expiry: datetime | None = None
def get_effective_root(self, sub_path: str | None) -> str:
"""
获取有效根路径。
:param sub_path: 子路径。
:return: 完整的有效路径。
"""
if sub_path:
return f"/{self.root.strip('/')}/{sub_path.strip('/')}".strip()
return f"/{self.root.strip('/')}".strip()
def _get_api_path(self, rel_path: str) -> str:
"""
将用户可见的相对路径转换为 Graph API 路径段。
:param rel_path: 相对路径。
:return: Graph API 路径段。
"""
full_path = self.get_effective_root(rel_path).strip('/')
if not full_path:
return ""
return f":/{full_path}"
async def _get_access_token(self) -> str:
"""
获取或刷新 access token。
:return: access token。
"""
if self._access_token and self._token_expiry and datetime.now(timezone.utc) < self._token_expiry:
return self._access_token
data = {
"client_id": self.client_id,
"client_secret": self.client_secret,
"refresh_token": self.refresh_token,
"grant_type": "refresh_token",
}
async with httpx.AsyncClient(timeout=20.0) as client:
resp = await client.post(MS_OAUTH_URL, data=data)
resp.raise_for_status()
token_data = resp.json()
self._access_token = token_data["access_token"]
self._token_expiry = datetime.now(
timezone.utc) + timedelta(seconds=token_data["expires_in"] - 300)
return self._access_token
async def _request(self, method: str, api_path_segment: str | None = None, *, full_url: str | None = None, **kwargs):
"""
向 Microsoft Graph API 发送请求。
:param method: HTTP 方法。
:param api_path_segment: API 路径段 (与 full_url 互斥)。
:param full_url: 完整的请求 URL (与 api_path_segment 互斥)。
:param kwargs: 其他请求参数。
:return: 响应对象。
"""
if not ((api_path_segment is not None) ^ (full_url is not None)):
raise ValueError("必须提供 api_path_segment 或 full_url 中的一个,且仅一个")
token = await self._get_access_token()
headers = {"Authorization": f"Bearer {token}"}
if "headers" in kwargs:
headers.update(kwargs.pop("headers"))
url = full_url if full_url else f"{MS_GRAPH_URL}/me/drive/root{api_path_segment}"
async with httpx.AsyncClient(timeout=60.0) as client:
resp = await client.request(method, url, headers=headers, **kwargs)
if resp.status_code == 401:
self._access_token = None
token = await self._get_access_token()
headers["Authorization"] = f"Bearer {token}"
resp = await client.request(method, url, headers=headers, **kwargs)
return resp
def _format_item(self, item: Dict) -> Dict:
"""
将 Graph API 返回的 item 格式化为统一的格式。
:param item: Graph API 返回的 item 字典。
:return: 格式化后的字典。
"""
is_dir = "folder" in item
return {
"name": item["name"],
"is_dir": is_dir,
"size": 0 if is_dir else item.get("size", 0),
"mtime": int(datetime.fromisoformat(item["lastModifiedDateTime"].replace("Z", "+00:00")).timestamp()),
"type": "dir" if is_dir else "file",
}
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
"""
列出目录内容。
由于 Graph API 不支持基于偏移($skip)的分页,此方法将获取所有项目,
:param root: 根路径 (在此适配器中未使用,通过配置的 root 确定)。
:param rel: 相对路径。
:param page_num: 页码。
:param page_size: 每页大小。
:param sort_by: 排序字段
:param sort_order: 排序顺序
:return: 文件/目录列表和总数。
"""
api_path = self._get_api_path(rel)
children_path = f"{api_path}:/children" if api_path else "/children"
all_items = []
params = {"$top": 999}
resp = await self._request("GET", api_path_segment=children_path, params=params)
while True:
if resp.status_code == 404 and not all_items:
return [], 0
resp.raise_for_status()
try:
data = resp.json()
except Exception as e:
raise IOError(f"解析 Graph API 响应失败: {e}") from e
all_items.extend(data.get("value", []))
next_link = data.get("@odata.nextLink")
if not next_link:
break
resp = await self._request("GET", full_url=next_link)
formatted_items = [self._format_item(item) for item in all_items]
# 排序
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
key = (not item["is_dir"],)
sort_field = sort_by.lower()
if sort_field == "name":
key += (item["name"].lower(),)
elif sort_field == "size":
key += (item["size"],)
elif sort_field == "mtime":
key += (item["mtime"],)
else:
key += (item["name"].lower(),)
return key
formatted_items.sort(key=get_sort_key, reverse=reverse)
total_count = len(formatted_items)
start_idx = (page_num - 1) * page_size
end_idx = start_idx + page_size
return formatted_items[start_idx:end_idx], total_count
async def read_file(self, root: str, rel: str) -> bytes:
"""
读取文件内容。
:param root: 根路径。
:param rel: 相对路径。
:return: 文件内容的字节流。
"""
api_path = self._get_api_path(rel)
if not api_path:
raise IsADirectoryError("不能将根目录作为文件读取")
resp = await self._request("GET", api_path_segment=f"{api_path}:/content")
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
return resp.content
async def write_file(self, root: str, rel: str, data: bytes):
"""
写入文件。
:param root: 根路径。
:param rel: 相对路径。
:param data: 文件内容的字节流。
"""
api_path = self._get_api_path(rel)
if not api_path:
raise ValueError("不能直接写入根路径")
resp = await self._request("PUT", api_path_segment=f"{api_path}:/content", content=data)
resp.raise_for_status()
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
"""
以流式方式写入文件。
:param root: 根路径。
:param rel: 相对路径。
:param data_iter: 文件内容的异步迭代器。
:return: 文件大小。
"""
api_path = self._get_api_path(rel)
if not api_path:
raise ValueError("不能直接写入根路径")
resp = await self._request("PUT", api_path_segment=f"{api_path}:/content", content=data_iter)
resp.raise_for_status()
return resp.json().get("size", 0)
async def mkdir(self, root: str, rel: str):
"""
创建目录。
:param root: 根路径。
:param rel: 相对路径。
"""
parent_path_str, new_dir_name = rel.rstrip(
'/').rsplit('/', 1) if '/' in rel.rstrip('/') else ('', rel)
parent_api_path = self._get_api_path(parent_path_str)
children_path = f"{parent_api_path}:/children" if parent_api_path else "/children"
payload = {
"name": new_dir_name,
"folder": {},
"@microsoft.graph.conflictBehavior": "fail" # 如果已存在则失败
}
resp = await self._request("POST", api_path_segment=children_path, json=payload)
resp.raise_for_status()
async def delete(self, root: str, rel: str):
"""
删除文件或目录。
:param root: 根路径。
:param rel: 相对路径。
"""
api_path = self._get_api_path(rel)
if not api_path:
raise ValueError("不能删除根目录")
resp = await self._request("DELETE", api_path_segment=api_path)
if resp.status_code not in (204, 404):
resp.raise_for_status()
async def move(self, root: str, src_rel: str, dst_rel: str):
"""
移动或重命名文件/目录。
:param root: 根路径。
:param src_rel: 源相对路径。
:param dst_rel: 目标相对路径。
"""
src_api_path = self._get_api_path(src_rel)
if not src_api_path:
raise ValueError("不能移动根目录")
dst_parent_rel, dst_name = dst_rel.rstrip(
'/').rsplit('/', 1) if '/' in dst_rel.rstrip('/') else ('', dst_rel)
dst_parent_api_path = self._get_api_path(dst_parent_rel)
# 获取父项目的 ID
parent_resp = await self._request("GET", api_path_segment=dst_parent_api_path)
parent_resp.raise_for_status()
parent_id = parent_resp.json()["id"]
payload = {
"parentReference": {"id": parent_id},
"name": dst_name
}
resp = await self._request("PATCH", api_path_segment=src_api_path, json=payload)
resp.raise_for_status()
async def rename(self, root: str, src_rel: str, dst_rel: str):
"""
重命名文件或目录。
在 Graph API 中,移动和重命名是同一个 PATCH 操作。
"""
await self.move(root, src_rel, dst_rel)
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
"""
复制文件或目录。
:param root: 根路径。
:param src_rel: 源相对路径。
:param dst_rel: 目标相对路径。
:param overwrite: 是否覆盖 (在此 API 中未直接使用)。
"""
src_api_path = self._get_api_path(src_rel)
if not src_api_path:
raise ValueError("不能复制根目录")
dst_parent_rel, dst_name = dst_rel.rstrip(
'/').rsplit('/', 1) if '/' in dst_rel.rstrip('/') else ('', dst_rel)
dst_parent_api_path = self._get_api_path(dst_parent_rel)
parent_resp = await self._request("GET", api_path_segment=dst_parent_api_path)
parent_resp.raise_for_status()
parent_id = parent_resp.json()["id"]
payload = {"parentReference": {"id": parent_id}, "name": dst_name}
copy_path = f"{src_api_path}:/copy"
resp = await self._request("POST", api_path_segment=copy_path, json=payload)
resp.raise_for_status()
async def stream_file(self, root: str, rel: str, range_header: str | None):
"""
流式传输文件(支持范围请求)。
:param root: 根路径。
:param rel: 相对路径。
:param range_header: HTTP Range 头。
:return: FastAPI StreamingResponse 对象。
"""
api_path = self._get_api_path(rel)
if not api_path:
raise IsADirectoryError("不能对目录进行流式传输")
resp = await self._request("GET", api_path_segment=api_path)
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
item_data = resp.json()
download_url = item_data.get("@microsoft.graph.downloadUrl")
if not download_url:
raise Exception("无法获取下载 URL")
file_size = item_data.get("size", 0)
content_type = item_data.get("file", {}).get(
"mimeType", "application/octet-stream")
start = 0
end = file_size - 1
status = 200
headers = {
"Accept-Ranges": "bytes",
"Content-Type": content_type,
"Content-Disposition": f"inline; filename=\"{item_data.get('name')}\""
}
if range_header and range_header.startswith("bytes="):
try:
part = range_header.removeprefix("bytes=")
s, e = part.split("-", 1)
if s.strip():
start = int(s)
if e.strip():
end = int(e)
if start >= file_size:
raise HTTPException(416, "Requested Range Not Satisfiable")
if end >= file_size:
end = file_size - 1
status = 206
except ValueError:
raise HTTPException(400, "Invalid Range header")
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
headers["Content-Length"] = str(end - start + 1)
else:
headers["Content-Length"] = str(file_size)
async def file_iterator():
nonlocal start, end
async with httpx.AsyncClient(timeout=60.0) as client:
req_headers = {'Range': f'bytes={start}-{end}'}
async with client.stream("GET", download_url, headers=req_headers) as stream_resp:
stream_resp.raise_for_status()
async for chunk in stream_resp.aiter_bytes():
yield chunk
return StreamingResponse(file_iterator(), status_code=status, headers=headers, media_type=content_type)
async def get_direct_download_response(self, root: str, rel: str):
if not self.enable_redirect_307:
return None
api_path = self._get_api_path(rel)
if not api_path:
raise IsADirectoryError("不能对目录进行直链重定向")
resp = await self._request("GET", api_path_segment=api_path)
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
item_data = resp.json()
download_url = item_data.get("@microsoft.graph.downloadUrl")
if not download_url:
return None
return Response(status_code=307, headers={"Location": download_url})
async def get_thumbnail(self, root: str, rel: str, size: str = "medium"):
"""
获取文件的缩略图。
:param root: 根路径。
:param rel: 相对路径。
:param size: 缩略图大小 (large, medium, small)。
:return: 缩略图内容的字节流,或在不支持时返回 None。
"""
api_path = self._get_api_path(rel)
if not api_path:
return None
thumb_path = f"{api_path}:/thumbnails/0/{size}"
try:
resp = await self._request("GET", api_path_segment=thumb_path)
if resp.status_code == 200:
thumb_data = resp.json()
async with httpx.AsyncClient(timeout=30.0) as client:
thumb_resp = await client.get(thumb_data['url'])
thumb_resp.raise_for_status()
return thumb_resp.content
elif resp.status_code == 404:
return None
else:
resp.raise_for_status()
except Exception:
return None
async def stat_file(self, root: str, rel: str):
"""
获取文件或目录的元数据。
:param root: 根路径。
:param rel: 相对路径。
:return: 格式化后的文件/目录信息。
"""
api_path = self._get_api_path(rel)
resp = await self._request("GET", api_path_segment=api_path)
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
return self._format_item(resp.json())
ADAPTER_TYPE = "onedrive"
CONFIG_SCHEMA = [
{"key": "client_id", "label": "Client ID", "type": "string", "required": True},
{"key": "client_secret", "label": "Client Secret",
"type": "password", "required": True},
{"key": "refresh_token", "label": "Refresh Token", "type": "password",
"required": True, "help_text": "可以通过运行 'python -m domain.adapters.providers.onedrive' 获取"},
{"key": "root", "label": "根目录 (Root Path)", "type": "string",
"required": False, "placeholder": "默认为根目录 /"},
{"key": "enable_direct_download_307", "label": "Enable 307 redirect download", "type": "boolean", "default": False},
]
def ADAPTER_FACTORY(rec): return OneDriveAdapter(rec)
+731
View File
@@ -0,0 +1,731 @@
from __future__ import annotations
import asyncio
import base64
import hashlib
import mimetypes
import os
import time
from typing import Dict, List, Tuple, Optional, AsyncIterator, Any
import httpx
from fastapi import HTTPException
from fastapi.responses import StreamingResponse
from models import StorageAdapter
from .base import BaseAdapter
# Quark 普通(UC)接口
API_BASE = "https://drive.quark.cn/1/clouddrive"
REFERER = "https://pan.quark.cn"
PR = "ucpro"
class QuarkAdapter:
"""夸克网盘(Cookie 模式)
- 使用浏览器导出的 Cookie 进行鉴权
- 通过 Quark/UC 的 clouddrive 接口实现:列目录、读写、分片上传、基础操作
- 根 FID 固定为 "0";路径解析通过名称遍历
"""
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config or {}
self.cookie: str = cfg.get("cookie") or cfg.get("Cookie")
self.root_fid: str = cfg.get("root_fid", "0")
def _as_bool(value: Any) -> bool:
if isinstance(value, bool):
return value
if isinstance(value, str):
return value.strip().lower() in {"1", "true", "yes", "on"}
return bool(value)
self.use_transcoding_address: bool = _as_bool(cfg.get("use_transcoding_address", False))
self.only_list_video_file: bool = _as_bool(cfg.get("only_list_video_file", False))
if not self.cookie:
raise ValueError("Quark 适配器需要 cookie 配置")
# 运行期缓存
self._dir_fid_cache: Dict[str, str] = {f"{self.root_fid}:": self.root_fid}
self._children_cache: Dict[str, List[Dict[str, Any]]] = {}
# UA 与超时
self._ua = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) quark-cloud-drive/2.5.20 Chrome/100.0.4896.160 "
"Electron/18.3.5.4-b478491100 Safari/537.36 Channel/pckk_other_ch"
)
self._timeout = 30.0
# -----------------
# 工具与通用请求
# -----------------
def get_effective_root(self, sub_path: str | None) -> str:
return self.root_fid
async def _request(
self,
method: str,
pathname: str,
*,
json: Any | None = None,
params: Dict[str, str] | None = None,
) -> Any:
headers = {
"Cookie": self._safe_cookie(self.cookie),
"Accept": "application/json, text/plain, */*",
"Referer": REFERER,
"User-Agent": self._ua,
}
query = {"pr": PR, "fr": "pc"}
if params:
query.update(params)
url = f"{API_BASE}{pathname}"
async with httpx.AsyncClient(timeout=self._timeout) as client:
resp = await client.request(method, url, headers=headers, params=query, json=json)
# 更新运行期 cookie(若返回 __puus/__pus
try:
for key in ("__puus", "__pus"):
v = resp.cookies.get(key)
if v:
# 简单替换/追加到 self.cookie
self._set_cookie_kv(key, v)
except Exception:
pass
# 解析业务状态
data = None
try:
data = resp.json()
except Exception:
resp.raise_for_status()
return resp
status = data.get("status")
code = data.get("code")
msg = data.get("message") or ""
if (status is not None and status >= 400) or (code is not None and code != 0):
raise HTTPException(502, detail=f"Quark error status={status} code={code} msg={msg}")
return data
def _set_cookie_kv(self, key: str, value: str):
# 将指定键值写入 self.cookie(粗略字符串处理)
parts = [p.strip() for p in (self.cookie or "").replace("\r", "").replace("\n", "").split(";") if p.strip()]
found = False
for i, p in enumerate(parts):
if p.startswith(key + "="):
parts[i] = f"{key}={value}"
found = True
break
if not found:
parts.append(f"{key}={value}")
self.cookie = "; ".join(parts)
def _sanitize_cookie(self, cookie: str) -> str:
if not cookie:
return ""
# 去除换行与前后空白
cookie = cookie.replace("\r", "").replace("\n", "").strip()
# 统一分号分隔并去除多余空格/空段
parts = [p.strip() for p in cookie.split(";") if p.strip()]
return "; ".join(parts)
def _safe_cookie(self, cookie: str) -> str:
s = self._sanitize_cookie(cookie)
# 仅保留可见 ASCII (0x20-0x7E)
s = "".join(ch for ch in s if 32 <= ord(ch) <= 126)
return s
# -----------------
# 列表与路径解析
# -----------------
def _map_file_item(self, it: Dict[str, Any]) -> Dict[str, Any]:
# Quark/UC 列表项:file=true 表示文件;false 表示目录
is_dir = not bool(it.get("file", False))
updated_at_ms = int(it.get("updated_at", 0) or 0)
name = it.get("file_name") or it.get("filename") or it.get("name")
return {
"fid": it.get("fid"),
"name": name,
"is_dir": is_dir,
"size": 0 if is_dir else int(it.get("size", 0) or 0),
"mtime": updated_at_ms // 1000 if updated_at_ms else 0,
"type": "dir" if is_dir else "file",
}
async def _list_children(self, parent_fid: str) -> List[Dict[str, Any]]:
if parent_fid in self._children_cache:
return self._children_cache[parent_fid]
files: List[Dict[str, Any]] = []
page = 1
size = 100
total = None
while True:
qp = {"pdir_fid": parent_fid, "_size": str(size), "_page": str(page), "_fetch_total": "1"}
data = await self._request("GET", "/file/sort", params=qp)
d = (data or {}).get("data", {})
meta = (data or {}).get("metadata", {})
page_files = d.get("list", [])
files.extend(page_files)
if total is None:
total = meta.get("_total") or meta.get("total") or 0
if page * size >= int(total):
break
page += 1
mapped = [self._map_file_item(x) for x in files if (not self.only_list_video_file) or (not x.get("file")) or (x.get("category") == 1)]
self._children_cache[parent_fid] = mapped
return mapped
def _dir_cache_key(self, base_fid: str, rel: str) -> str:
return f"{base_fid}:{rel.strip('/')}"
async def _resolve_dir_fid_from(self, base_fid: str, rel: str) -> str:
key = rel.strip("/")
cache_key = self._dir_cache_key(base_fid, key)
if cache_key in self._dir_fid_cache:
return self._dir_fid_cache[cache_key]
if key == "":
self._dir_fid_cache[cache_key] = base_fid
return base_fid
parent_fid = base_fid
path_so_far = []
for seg in key.split("/"):
if seg == "":
continue
path_so_far.append(seg)
cache_key = self._dir_cache_key(base_fid, "/".join(path_so_far))
cached = self._dir_fid_cache.get(cache_key)
if cached:
parent_fid = cached
continue
children = await self._list_children(parent_fid)
found = next((c for c in children if c["is_dir"] and c["name"] == seg), None)
if not found:
raise FileNotFoundError(f"Directory not found: {seg}")
parent_fid = found["fid"]
self._dir_fid_cache[cache_key] = parent_fid
return parent_fid
async def _find_child(self, parent_fid: str, name: str) -> Optional[Dict[str, Any]]:
children = await self._list_children(parent_fid)
for it in children:
if it["name"] == name:
return it
return None
def _invalidate_children_cache(self, parent_fid: str):
if parent_fid in self._children_cache:
try:
del self._children_cache[parent_fid]
except Exception:
pass
# -----------------
# 目录与文件列表
# -----------------
async def list_dir(
self,
root: str,
rel: str,
page_num: int = 1,
page_size: int = 50,
sort_by: str = "name",
sort_order: str = "asc",
) -> Tuple[List[Dict], int]:
base_fid = root or self.root_fid
fid = await self._resolve_dir_fid_from(base_fid, rel)
items = await self._list_children(fid)
# 排序,目录优先
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
key = (not item["is_dir"],)
sf = sort_by.lower()
if sf == "name":
key += (item["name"].lower(),)
elif sf == "size":
key += (item["size"],)
elif sf == "mtime":
key += (item["mtime"],)
else:
key += (item["name"].lower(),)
return key
items.sort(key=get_sort_key, reverse=reverse)
total = len(items)
start = (page_num - 1) * page_size
end = start + page_size
return items[start:end], total
# -----------------
# 下载与流式下载
# -----------------
async def _get_download_url(self, fid: str) -> str:
data = await self._request("POST", "/file/download", json={"fids": [fid]})
arr = (data or {}).get("data", [])
if not arr:
raise HTTPException(502, detail="No download data returned by Quark")
url = arr[0].get("download_url") or arr[0].get("DownloadUrl")
if not url:
raise HTTPException(502, detail="No download_url returned by Quark")
return url
async def _get_transcoding_url(self, fid: str) -> Optional[str]:
try:
payload = {"fid": fid, "resolutions": "low,normal,high,super,2k,4k", "supports": "fmp4_av,m3u8,dolby_vision"}
data = await self._request("POST", "/file/v2/play/project", json=payload)
lst = (data or {}).get("data", {}).get("video_list", [])
for item in lst:
vi = item.get("video_info") or {}
url = vi.get("url")
if url:
return url
except Exception:
return None
return None
def _is_video_name(self, name: str) -> bool:
mime, _ = mimetypes.guess_type(name)
return bool(mime and mime.startswith("video/"))
def _download_headers(self) -> Dict[str, str]:
return {"Cookie": self._safe_cookie(self.cookie), "User-Agent": self._ua, "Referer": REFERER}
async def read_file(self, root: str, rel: str) -> bytes:
if not rel or rel.endswith("/"):
raise IsADirectoryError("Path is a directory")
parent = rel.rsplit("/", 1)[0] if "/" in rel else ""
name = rel.rsplit("/", 1)[-1]
base_fid = root or self.root_fid
parent_fid = await self._resolve_dir_fid_from(base_fid, parent)
it = await self._find_child(parent_fid, name)
if not it or it["is_dir"]:
raise FileNotFoundError(rel)
url = await self._get_download_url(it["fid"])
headers = self._download_headers()
async with httpx.AsyncClient(timeout=None, follow_redirects=True) as client:
resp = await client.get(url, headers=headers)
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
return resp.content
async def stream_file(self, root: str, rel: str, range_header: str | None):
if not rel or rel.endswith("/"):
raise IsADirectoryError("Path is a directory")
parent = rel.rsplit("/", 1)[0] if "/" in rel else ""
name = rel.rsplit("/", 1)[-1]
base_fid = root or self.root_fid
parent_fid = await self._resolve_dir_fid_from(base_fid, parent)
it = await self._find_child(parent_fid, name)
if not it or it["is_dir"]:
raise FileNotFoundError(rel)
url = await self._get_download_url(it["fid"])
if self.use_transcoding_address and self._is_video_name(name):
tr = await self._get_transcoding_url(it["fid"])
if tr:
url = tr
dl_headers = self._download_headers()
# 预获取大小/是否支持范围
total_size: Optional[int] = None
async with httpx.AsyncClient(timeout=self._timeout, follow_redirects=True) as client:
try:
head_resp = await client.head(url, headers=dl_headers)
if head_resp.status_code == 200:
cl = head_resp.headers.get("Content-Length")
if cl and cl.isdigit():
total_size = int(cl)
except Exception:
pass
mime, _ = mimetypes.guess_type(rel)
content_type = mime or "application/octet-stream"
# 解析 Range
start = 0
end: Optional[int] = None
status_code = 200
if range_header and range_header.startswith("bytes="):
status_code = 206
part = range_header.split("=", 1)[1]
s, e = part.split("-", 1)
if s.strip():
start = int(s)
if e.strip():
end = int(e)
if total_size is not None and end is None and status_code == 206:
end = total_size - 1
if end is not None and total_size is not None and end >= total_size:
end = total_size - 1
if total_size is not None and start >= total_size:
raise HTTPException(416, detail="Requested Range Not Satisfiable")
resp_headers: Dict[str, str] = {"Accept-Ranges": "bytes", "Content-Type": content_type}
if status_code == 206 and total_size is not None and end is not None:
resp_headers["Content-Range"] = f"bytes {start}-{end}/{total_size}"
resp_headers["Content-Length"] = str(end - start + 1)
elif total_size is not None:
resp_headers["Content-Length"] = str(total_size)
async def iterator():
headers = dict(dl_headers)
if status_code == 206 and end is not None:
headers["Range"] = f"bytes={start}-{end}"
async with httpx.AsyncClient(timeout=None, follow_redirects=True) as client:
async with client.stream("GET", url, headers=headers) as resp:
if resp.status_code in (404, 416):
await resp.aclose()
raise HTTPException(resp.status_code, detail="Upstream not available")
async for chunk in resp.aiter_bytes():
if chunk:
yield chunk
return StreamingResponse(iterator(), status_code=status_code, headers=resp_headers, media_type=content_type)
# -----------------
# 上传(大文件分片)
# -----------------
@staticmethod
def _md5_hex(b: bytes) -> str:
return hashlib.md5(b).hexdigest()
@staticmethod
def _sha1_hex(b: bytes) -> str:
return hashlib.sha1(b).hexdigest()
def _guess_mime(self, name: str) -> str:
mime, _ = mimetypes.guess_type(name)
return mime or "application/octet-stream"
async def _upload_pre(self, filename: str, size: int, parent_fid: str) -> Dict[str, Any]:
now_ms = int(time.time() * 1000)
body = {
"ccp_hash_update": True,
"dir_name": "",
"file_name": filename,
"format_type": self._guess_mime(filename),
"l_created_at": now_ms,
"l_updated_at": now_ms,
"pdir_fid": parent_fid,
"size": size,
}
data = await self._request("POST", "/file/upload/pre", json=body)
return data
async def write_file(self, root: str, rel: str, data: bytes):
async def gen():
yield data
return await self.write_file_stream(root, rel, gen())
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
if not rel or rel.endswith("/"):
raise HTTPException(400, detail="Invalid file path")
parent = rel.rsplit("/", 1)[0] if "/" in rel else ""
name = rel.rsplit("/", 1)[-1]
base_fid = root or self.root_fid
parent_fid = await self._resolve_dir_fid_from(base_fid, parent)
# 将数据落盘到临时文件,同时计算 MD5/SHA1
import tempfile
md5 = hashlib.md5()
sha1 = hashlib.sha1()
total = 0
with tempfile.NamedTemporaryFile(delete=False) as tf:
tmp_path = tf.name
try:
async for chunk in data_iter:
if not chunk:
continue
total += len(chunk)
md5.update(chunk)
sha1.update(chunk)
tf.write(chunk)
finally:
tf.flush()
md5_hex = md5.hexdigest()
sha1_hex = sha1.hexdigest()
# 预上传,拿到上传信息
pre_resp = await self._upload_pre(name, total, parent_fid)
pre_data = pre_resp.get("data", {})
# hash 秒传
hash_body = {"md5": md5_hex, "sha1": sha1_hex, "task_id": pre_data.get("task_id")}
hash_resp = await self._request("POST", "/file/update/hash", json=hash_body)
if (hash_resp.get("data") or {}).get("finish") is True:
try:
os.unlink(tmp_path)
except Exception:
pass
# 刷新父目录缓存
self._invalidate_children_cache(parent_fid)
return total
# 分片上传
part_size = int((pre_resp.get("metadata") or {}).get("part_size") or 0)
if part_size <= 0:
raise HTTPException(502, detail="Invalid part_size from Quark")
bucket = pre_data.get("bucket")
obj_key = pre_data.get("obj_key")
upload_id = pre_data.get("upload_id")
upload_url = pre_data.get("upload_url")
if not (bucket and obj_key and upload_id and upload_url):
raise HTTPException(502, detail="Upload pre missing fields")
# 计算 host 与基础 URL
try:
upload_host = upload_url.split("://", 1)[1]
except Exception:
upload_host = upload_url
base_url = f"https://{bucket}.{upload_host}/{obj_key}"
# 分片循环
etags: List[str] = []
oss_ua = "aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit"
async with httpx.AsyncClient(timeout=None, follow_redirects=True) as client:
with open(tmp_path, "rb") as rf:
part_number = 1
left = total
while left > 0:
sz = min(part_size, left)
data_bytes = rf.read(sz)
if len(data_bytes) != sz:
raise IOError("Failed to read part bytes")
now_str = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
# 申请签名
auth_meta = (
"PUT\n\n"
f"{self._guess_mime(name)}\n"
f"{now_str}\n"
f"x-oss-date:{now_str}\n"
f"x-oss-user-agent:{oss_ua}\n"
f"/{bucket}/{obj_key}?partNumber={part_number}&uploadId={upload_id}"
)
auth_req_body = {"auth_info": pre_data.get("auth_info"), "auth_meta": auth_meta, "task_id": pre_data.get("task_id")}
auth_resp = await self._request("POST", "/file/upload/auth", json=auth_req_body)
auth_key = (auth_resp.get("data") or {}).get("auth_key")
if not auth_key:
raise HTTPException(502, detail="upload/auth missing auth_key")
put_headers = {
"Authorization": auth_key,
"Content-Type": self._guess_mime(name),
"Referer": REFERER + "/",
"x-oss-date": now_str,
"x-oss-user-agent": oss_ua,
}
put_url = f"{base_url}?partNumber={part_number}&uploadId={upload_id}"
put_resp = await client.put(put_url, headers=put_headers, content=data_bytes)
if put_resp.status_code != 200:
raise HTTPException(502, detail=f"Upload part failed status={put_resp.status_code} text={put_resp.text}")
etag = put_resp.headers.get("Etag", "")
etags.append(etag)
left -= sz
part_number += 1
# 组合 commit xml
parts_xml = [f"<Part>\n<PartNumber>{i+1}</PartNumber>\n<ETag>{etags[i]}</ETag>\n</Part>\n" for i in range(len(etags))]
body_xml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<CompleteMultipartUpload>\n" + "".join(parts_xml) + "</CompleteMultipartUpload>"
content_md5 = base64.b64encode(hashlib.md5(body_xml.encode("utf-8")).digest()).decode("ascii")
callback = pre_data.get("callback") or {}
try:
import json as _json
callback_b64 = base64.b64encode(_json.dumps(callback).encode("utf-8")).decode("ascii")
except Exception:
callback_b64 = ""
now_str = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
auth_meta_commit = (
"POST\n"
f"{content_md5}\n"
"application/xml\n"
f"{now_str}\n"
f"x-oss-callback:{callback_b64}\n"
f"x-oss-date:{now_str}\n"
f"x-oss-user-agent:{oss_ua}\n"
f"/{bucket}/{obj_key}?uploadId={upload_id}"
)
auth_commit_resp = await self._request("POST", "/file/upload/auth", json={"auth_info": pre_data.get("auth_info"), "auth_meta": auth_meta_commit, "task_id": pre_data.get("task_id")})
auth_key_commit = (auth_commit_resp.get("data") or {}).get("auth_key")
if not auth_key_commit:
raise HTTPException(502, detail="upload/auth(commit) missing auth_key")
async with httpx.AsyncClient(timeout=None, follow_redirects=True) as client:
commit_headers = {
"Authorization": auth_key_commit,
"Content-MD5": content_md5,
"Content-Type": "application/xml",
"Referer": REFERER + "/",
"x-oss-callback": callback_b64,
"x-oss-date": now_str,
"x-oss-user-agent": oss_ua,
}
commit_url = f"{base_url}?uploadId={upload_id}"
r = await client.post(commit_url, headers=commit_headers, content=body_xml.encode("utf-8"))
if r.status_code != 200:
raise HTTPException(502, detail=f"Upload commit failed status={r.status_code} text={r.text}")
# finish
await self._request("POST", "/file/upload/finish", json={"obj_key": obj_key, "task_id": pre_data.get("task_id")})
# 端合并存在轻微延迟,等待再刷新缓存
try:
await asyncio.sleep(1.0)
except Exception:
pass
try:
os.unlink(tmp_path)
except Exception:
pass
# 失效父目录缓存,确保后续列表可见
self._invalidate_children_cache(parent_fid)
return total
# -----------------
# 基本文件操作
# -----------------
async def mkdir(self, root: str, rel: str):
if not rel or rel == "/":
raise HTTPException(400, detail="Cannot create root")
parent = rel.rstrip("/")
parent_rel, name = (parent.rsplit("/", 1) if "/" in parent else ("", parent))
if not name:
raise HTTPException(400, detail="Invalid directory name")
pdir = await self._resolve_dir_fid_from(root or self.root_fid, parent_rel)
await self._request("POST", "/file", json={"dir_init_lock": False, "dir_path": "", "file_name": name, "pdir_fid": pdir})
self._invalidate_children_cache(pdir)
async def delete(self, root: str, rel: str):
# 解析对象 fid + 父目录,用于失效缓存
base_fid = root or self.root_fid
if rel == "" or rel.endswith("/"):
parent_rel = rel.rstrip("/")
target_fid = await self._resolve_dir_fid_from(base_fid, parent_rel)
parent_of_target = await self._resolve_dir_fid_from(base_fid, (parent_rel.rsplit("/", 1)[0] if "/" in parent_rel else ""))
else:
parent_rel, name = (rel.rsplit("/", 1) if "/" in rel else ("", rel))
parent_of_target = await self._resolve_dir_fid_from(base_fid, parent_rel)
it = await self._find_child(parent_of_target, name)
if not it:
return
target_fid = it["fid"]
await self._request("POST", "/file/delete", json={"action_type": 1, "exclude_fids": [], "filelist": [target_fid]})
self._invalidate_children_cache(parent_of_target)
async def move(self, root: str, src_rel: str, dst_rel: str):
# 支持跨目录与重命名:先移动到父目录,后重命名(若需要)
src_parent_rel, src_name = (src_rel.rsplit("/", 1) if "/" in src_rel else ("", src_rel))
dst_parent_rel, dst_name = (dst_rel.rsplit("/", 1) if "/" in dst_rel else ("", dst_rel))
base_fid = root or self.root_fid
src_parent_fid = await self._resolve_dir_fid_from(base_fid, src_parent_rel)
obj = await self._find_child(src_parent_fid, src_name)
if not obj:
raise FileNotFoundError(src_rel)
dst_parent_fid = await self._resolve_dir_fid_from(base_fid, dst_parent_rel)
if src_parent_fid != dst_parent_fid:
await self._request("POST", "/file/move", json={"action_type": 1, "exclude_fids": [], "filelist": [obj["fid"]], "to_pdir_fid": dst_parent_fid})
self._invalidate_children_cache(src_parent_fid)
self._invalidate_children_cache(dst_parent_fid)
if obj["name"] != dst_name:
await self._request("POST", "/file/rename", json={"fid": obj["fid"], "file_name": dst_name})
self._invalidate_children_cache(dst_parent_fid)
async def rename(self, root: str, src_rel: str, dst_rel: str):
src_parent_rel, src_name = (src_rel.rsplit("/", 1) if "/" in src_rel else ("", src_rel))
base_fid = root or self.root_fid
src_parent_fid = await self._resolve_dir_fid_from(base_fid, src_parent_rel)
obj = await self._find_child(src_parent_fid, src_name)
if not obj:
raise FileNotFoundError(src_rel)
dst_name = dst_rel.rsplit("/", 1)[-1]
await self._request("POST", "/file/rename", json={"fid": obj["fid"], "file_name": dst_name})
self._invalidate_children_cache(src_parent_fid)
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
raise NotImplementedError("QuarkOpen does not support copy via open API")
# -----------------
# STAT / EXISTS / 辅助
# -----------------
async def stat_file(self, root: str, rel: str):
# 通过父目录列表获取元数据
base_fid = root or self.root_fid
if rel == "" or rel.endswith("/"):
# 目录
fid = await self._resolve_dir_fid_from(base_fid, rel.rstrip("/"))
return {"name": rel.rstrip("/").split("/")[-1] if rel else "", "is_dir": True, "size": 0, "mtime": 0, "type": "dir", "fid": fid}
parent_rel, name = (rel.rsplit("/", 1) if "/" in rel else ("", rel))
parent_fid = await self._resolve_dir_fid_from(base_fid, parent_rel)
it = await self._find_child(parent_fid, name)
if not it:
raise FileNotFoundError(rel)
return it
async def exists(self, root: str, rel: str) -> bool:
try:
base_fid = root or self.root_fid
if rel == "" or rel.endswith("/"):
await self._resolve_dir_fid_from(base_fid, rel.rstrip("/"))
return True
parent_rel, name = (rel.rsplit("/", 1) if "/" in rel else ("", rel))
parent_fid = await self._resolve_dir_fid_from(base_fid, parent_rel)
it = await self._find_child(parent_fid, name)
return it is not None
except FileNotFoundError:
return False
async def stat_path(self, root: str, rel: str):
# 用于 move/copy 前的预检查调试
try:
base_fid = root or self.root_fid
if rel == "" or rel.endswith("/"):
fid = await self._resolve_dir_fid_from(base_fid, rel.rstrip("/"))
return {"exists": True, "is_dir": True, "path": rel, "fid": fid}
parent_rel, name = (rel.rsplit("/", 1) if "/" in rel else ("", rel))
parent_fid = await self._resolve_dir_fid_from(base_fid, parent_rel)
it = await self._find_child(parent_fid, name)
if it:
return {"exists": True, "is_dir": it["is_dir"], "path": rel, "fid": it["fid"]}
return {"exists": False, "is_dir": None, "path": rel}
except FileNotFoundError:
return {"exists": False, "is_dir": None, "path": rel}
async def _resolve_target_fid(self, rel: str, *, base_fid: Optional[str] = None) -> str:
base = base_fid or self.root_fid
if rel == "" or rel.endswith("/"):
return await self._resolve_dir_fid_from(base, rel.rstrip("/"))
parent_rel, name = (rel.rsplit("/", 1) if "/" in rel else ("", rel))
parent_fid = await self._resolve_dir_fid_from(base, parent_rel)
it = await self._find_child(parent_fid, name)
if not it:
raise FileNotFoundError(rel)
return it["fid"]
ADAPTER_TYPE = "quark"
CONFIG_SCHEMA = [
{"key": "cookie", "label": "Cookie", "type": "password", "required": True, "placeholder": "从 pan.quark.cn 复制"},
{"key": "root_fid", "label": "根 FID", "type": "string", "required": False, "default": "0"},
{"key": "use_transcoding_address", "label": "视频转码直链", "type": "boolean", "required": False, "default": False},
{"key": "only_list_video_file", "label": "仅列出视频文件", "type": "boolean", "required": False, "default": False},
]
def ADAPTER_FACTORY(rec: StorageAdapter) -> BaseAdapter:
return QuarkAdapter(rec)
+348
View File
@@ -0,0 +1,348 @@
from __future__ import annotations
import asyncio
import mimetypes
from datetime import datetime
from typing import List, Dict, Tuple, AsyncIterator
from urllib.parse import quote
import aioboto3
from botocore.exceptions import ClientError
from fastapi import HTTPException
from fastapi.responses import StreamingResponse
from models import StorageAdapter
class S3Adapter:
"""S3 兼容对象存储适配器"""
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config
self.bucket_name = cfg.get("bucket_name")
self.aws_access_key_id = cfg.get("access_key_id")
self.aws_secret_access_key = cfg.get("secret_access_key")
self.region_name = cfg.get("region_name")
self.endpoint_url = cfg.get("endpoint_url")
self.root = cfg.get("root", "").strip("/")
if not all([self.bucket_name, self.aws_access_key_id, self.aws_secret_access_key]):
raise ValueError(
"S3 适配器需要 bucket_name, access_key_id, 和 secret_access_key")
self.session = aioboto3.Session(
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
region_name=self.region_name,
)
def get_effective_root(self, sub_path: str | None) -> str:
"""获取 S3 中的有效根路径 (key prefix)"""
if sub_path:
return f"{self.root}/{sub_path.strip('/')}".strip("/")
return self.root
def _get_s3_key(self, rel_path: str) -> str:
"""将相对路径转换为 S3 key"""
rel_path = rel_path.strip("/")
if self.root:
return f"{self.root}/{rel_path}"
return rel_path
def _get_client(self):
return self.session.client("s3", endpoint_url=self.endpoint_url)
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
prefix = self._get_s3_key(rel)
if prefix and not prefix.endswith("/"):
prefix += "/"
all_items = []
async with self._get_client() as s3:
paginator = s3.get_paginator("list_objects_v2")
async for result in paginator.paginate(Bucket=self.bucket_name, Prefix=prefix, Delimiter="/"):
# 添加子目录
for common_prefix in result.get("CommonPrefixes", []):
dir_name = common_prefix.get(
"Prefix").removeprefix(prefix).strip("/")
if dir_name:
all_items.append({
"name": dir_name,
"is_dir": True,
"size": 0,
"mtime": 0,
"type": "dir",
})
# 添加文件
for content in result.get("Contents", []):
file_key = content.get("Key")
if file_key == prefix: # 忽略目录本身
continue
file_name = file_key.removeprefix(prefix)
if file_name:
all_items.append({
"name": file_name,
"is_dir": False,
"size": content.get("Size", 0),
"mtime": int(content.get("LastModified", datetime.now()).timestamp()),
"type": "file",
})
# 在内存中排序和分页
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
key = (not item["is_dir"],)
sort_field = sort_by.lower()
if sort_field == "name":
key += (item["name"].lower(),)
elif sort_field == "size":
key += (item["size"],)
elif sort_field == "mtime":
key += (item["mtime"],)
else:
key += (item["name"].lower(),)
return key
all_items.sort(key=get_sort_key, reverse=reverse)
total_count = len(all_items)
start_idx = (page_num - 1) * page_size
end_idx = start_idx + page_size
return all_items[start_idx:end_idx], total_count
async def read_file(self, root: str, rel: str) -> bytes:
key = self._get_s3_key(rel)
async with self._get_client() as s3:
try:
resp = await s3.get_object(Bucket=self.bucket_name, Key=key)
return await resp["Body"].read()
except ClientError as e:
if e.response["Error"]["Code"] == "NoSuchKey":
raise FileNotFoundError(rel)
raise
async def write_file(self, root: str, rel: str, data: bytes):
key = self._get_s3_key(rel)
async with self._get_client() as s3:
await s3.put_object(Bucket=self.bucket_name, Key=key, Body=data)
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
key = self._get_s3_key(rel)
MIN_PART_SIZE = 5 * 1024 * 1024
async with self._get_client() as s3:
mpu = await s3.create_multipart_upload(Bucket=self.bucket_name, Key=key)
upload_id = mpu['UploadId']
parts = []
part_number = 1
total_size = 0
buffer = bytearray()
try:
async for chunk in data_iter:
if not chunk:
continue
buffer.extend(chunk)
while len(buffer) >= MIN_PART_SIZE:
part_data = buffer[:MIN_PART_SIZE]
del buffer[:MIN_PART_SIZE]
part = await s3.upload_part(
Bucket=self.bucket_name,
Key=key,
PartNumber=part_number,
UploadId=upload_id,
Body=part_data
)
parts.append({'PartNumber': part_number, 'ETag': part['ETag']})
total_size += len(part_data)
part_number += 1
if buffer:
part = await s3.upload_part(
Bucket=self.bucket_name,
Key=key,
PartNumber=part_number,
UploadId=upload_id,
Body=bytes(buffer)
)
parts.append({'PartNumber': part_number, 'ETag': part['ETag']})
total_size += len(buffer)
await s3.complete_multipart_upload(
Bucket=self.bucket_name,
Key=key,
UploadId=upload_id,
MultipartUpload={'Parts': parts}
)
except Exception as e:
await s3.abort_multipart_upload(
Bucket=self.bucket_name,
Key=key,
UploadId=upload_id
)
raise IOError(f"S3 stream upload failed: {e}") from e
return total_size
async def mkdir(self, root: str, rel: str):
key = self._get_s3_key(rel)
if not key.endswith("/"):
key += "/"
async with self._get_client() as s3:
await s3.put_object(Bucket=self.bucket_name, Key=key, Body=b"")
async def delete(self, root: str, rel: str):
key = self._get_s3_key(rel)
async with self._get_client() as s3:
is_dir_like = False
try:
head = await s3.head_object(Bucket=self.bucket_name, Key=key)
if head['ContentLength'] == 0 and key.endswith('/'):
is_dir_like = True
except ClientError as e:
if e.response['Error']['Code'] != '404':
raise
# 如果是目录,删除目录下的所有对象
if is_dir_like or not await self.stat_file(root, rel):
dir_key = key if key.endswith('/') else key + '/'
paginator = s3.get_paginator("list_objects_v2")
objects_to_delete = []
async for result in paginator.paginate(Bucket=self.bucket_name, Prefix=dir_key):
for content in result.get("Contents", []):
objects_to_delete.append({"Key": content["Key"]})
if objects_to_delete:
await s3.delete_objects(Bucket=self.bucket_name, Delete={"Objects": objects_to_delete})
# 如果是文件,直接删除
else:
await s3.delete_object(Bucket=self.bucket_name, Key=key)
async def move(self, root: str, src_rel: str, dst_rel: str):
await self.copy(root, src_rel, dst_rel, overwrite=True)
await self.delete(root, src_rel)
async def rename(self, root: str, src_rel: str, dst_rel: str):
await self.move(root, src_rel, dst_rel)
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
src_key = self._get_s3_key(src_rel)
dst_key = self._get_s3_key(dst_rel)
async with self._get_client() as s3:
if not overwrite:
try:
await s3.head_object(Bucket=self.bucket_name, Key=dst_key)
raise FileExistsError(dst_rel)
except ClientError as e:
if e.response["Error"]["Code"] != "404":
raise
copy_source = {"Bucket": self.bucket_name, "Key": src_key}
await s3.copy_object(CopySource=copy_source, Bucket=self.bucket_name, Key=dst_key)
async def stat_file(self, root: str, rel: str):
key = self._get_s3_key(rel)
async with self._get_client() as s3:
try:
head = await s3.head_object(Bucket=self.bucket_name, Key=key)
return {
"name": rel.split("/")[-1],
"is_dir": False,
"size": head["ContentLength"],
"mtime": int(head["LastModified"].timestamp()),
"type": "file",
}
except ClientError as e:
if e.response["Error"]["Code"] == "404":
# 检查是否为一个 "目录"
dir_key = key if key.endswith('/') else key + '/'
resp = await s3.list_objects_v2(Bucket=self.bucket_name, Prefix=dir_key, MaxKeys=1)
if resp.get('KeyCount', 0) > 0:
return {
"name": rel.split("/")[-1],
"is_dir": True,
"size": 0,
"mtime": 0,
"type": "dir",
}
raise FileNotFoundError(rel)
raise
async def stream_file(self, root: str, rel: str, range_header: str | None):
key = self._get_s3_key(rel)
async with self._get_client() as s3:
try:
head = await s3.head_object(Bucket=self.bucket_name, Key=key)
file_size = head["ContentLength"]
content_type = head.get("ContentType", mimetypes.guess_type(key)[
0] or "application/octet-stream")
except ClientError as e:
if e.response["Error"]["Code"] == "404":
raise HTTPException(
status_code=404, detail="File not found")
raise
start = 0
end = file_size - 1
status = 200
headers = {
"Accept-Ranges": "bytes",
"Content-Type": content_type,
"Content-Length": str(file_size),
"Content-Disposition": f"inline; filename=\"{quote(rel.split('/')[-1])}\""
}
if range_header:
range_val = range_header.strip().partition("=")[2]
s, _, e = range_val.partition("-")
try:
start = int(s) if s else 0
end = int(e) if e else file_size - 1
if start >= file_size or end >= file_size or start > end:
raise HTTPException(
status_code=416, detail="Requested Range Not Satisfiable")
status = 206
headers["Content-Length"] = str(end - start + 1)
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
except ValueError:
raise HTTPException(
status_code=400, detail="Invalid Range header")
range_arg = f"bytes={start}-{end}"
async def iterator():
try:
resp = await s3.get_object(Bucket=self.bucket_name, Key=key, Range=range_arg)
body = resp["Body"]
while chunk := await body.read(65536):
yield chunk
except Exception as e:
raise
return StreamingResponse(iterator(), status_code=status, headers=headers, media_type=content_type)
ADAPTER_TYPE = "s3"
CONFIG_SCHEMA = [
{"key": "bucket_name", "label": "Bucket 名称",
"type": "string", "required": True},
{"key": "access_key_id", "label": "Access Key ID",
"type": "string", "required": True},
{"key": "secret_access_key", "label": "Secret Access Key",
"type": "password", "required": True},
{"key": "region_name", "label": "区域 (Region)", "type": "string",
"required": False, "placeholder": "例如 us-east-1"},
{"key": "endpoint_url", "label": "Endpoint URL", "type": "string",
"required": False, "placeholder": "对于 S3 兼容存储, 例如 https://minio.example.com"},
{"key": "root", "label": "根路径 (Root Path)", "type": "string",
"required": False, "placeholder": "在 bucket 内的路径前缀"},
]
def ADAPTER_FACTORY(rec): return S3Adapter(rec)
+440
View File
@@ -0,0 +1,440 @@
from __future__ import annotations
import asyncio
import mimetypes
import stat as statmod
from typing import List, Dict, Tuple, AsyncIterator, Optional
from fastapi import HTTPException
from fastapi.responses import StreamingResponse
import paramiko
from models import StorageAdapter
def _join_remote(root: str, rel: str) -> str:
root = (root or "/").rstrip("/") or "/"
rel = (rel or "").lstrip("/")
if not rel:
return root
return f"{root}/{rel}"
class SFTPAdapter:
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config
self.host: str = cfg.get("host")
self.port: int = int(cfg.get("port", 22))
self.username: str | None = cfg.get("username")
self.password: str | None = cfg.get("password")
self.timeout: int = int(cfg.get("timeout", 15))
self.root_path: str = cfg.get("root") # 必填
self.allow_unknown_host: bool = bool(cfg.get("allow_unknown_host", True))
if not self.host:
raise ValueError("SFTP adapter requires 'host'")
if not self.username or not self.password:
raise ValueError("SFTP adapter requires 'username' and 'password'")
if not self.root_path:
raise ValueError("SFTP adapter requires 'root'")
def get_effective_root(self, sub_path: str | None) -> str:
base = self.root_path.rstrip("/") or "/"
if sub_path:
return _join_remote(base, sub_path)
return base
def _connect(self) -> paramiko.SFTPClient:
ssh = paramiko.SSHClient()
if self.allow_unknown_host:
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(
hostname=self.host,
port=self.port,
username=self.username,
password=self.password,
timeout=self.timeout,
allow_agent=False,
look_for_keys=False,
)
return ssh.open_sftp()
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
path = _join_remote(root, rel)
def _do_list() -> List[Dict]:
sftp = self._connect()
try:
attrs = sftp.listdir_attr(path)
entries: List[Dict] = []
for a in attrs:
name = a.filename
is_dir = statmod.S_ISDIR(a.st_mode)
entries.append({
"name": name,
"is_dir": is_dir,
"size": 0 if is_dir else int(a.st_size or 0),
"mtime": int(a.st_mtime or 0),
"type": "dir" if is_dir else "file",
})
return entries
finally:
try:
sftp.close()
except Exception:
pass
entries = await asyncio.to_thread(_do_list)
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
key = (not item["is_dir"],)
f = sort_by.lower()
if f == "name":
key += (item["name"].lower(),)
elif f == "size":
key += (item.get("size", 0),)
elif f == "mtime":
key += (item.get("mtime", 0),)
else:
key += (item["name"].lower(),)
return key
entries.sort(key=get_sort_key, reverse=reverse)
total = len(entries)
start = (page_num - 1) * page_size
end = start + page_size
return entries[start:end], total
async def read_file(self, root: str, rel: str) -> bytes:
path = _join_remote(root, rel)
def _do_read() -> bytes:
sftp = self._connect()
try:
with sftp.open(path, "rb") as f:
return f.read()
except FileNotFoundError:
raise
except IOError as e:
if getattr(e, "errno", None) == 2:
raise FileNotFoundError(rel)
raise
finally:
try:
sftp.close()
except Exception:
pass
return await asyncio.to_thread(_do_read)
async def write_file(self, root: str, rel: str, data: bytes):
path = _join_remote(root, rel)
def _ensure_dirs(sftp: paramiko.SFTPClient, dir_path: str):
parts = [p for p in dir_path.strip("/").split("/") if p]
cur = "/"
for p in parts:
cur = _join_remote(cur, p)
try:
sftp.mkdir(cur)
except IOError:
# likely exists
pass
def _do_write():
sftp = self._connect()
try:
parent = "/" if "/" not in path.strip("/") else path.rsplit("/", 1)[0]
_ensure_dirs(sftp, parent)
with sftp.open(path, "wb") as f:
f.write(data)
finally:
try:
sftp.close()
except Exception:
pass
await asyncio.to_thread(_do_write)
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
buf = bytearray()
async for chunk in data_iter:
if chunk:
buf.extend(chunk)
await self.write_file(root, rel, bytes(buf))
return len(buf)
async def mkdir(self, root: str, rel: str):
path = _join_remote(root, rel)
def _do_mkdir():
sftp = self._connect()
try:
parts = [p for p in path.strip("/").split("/") if p]
cur = "/"
for p in parts:
cur = _join_remote(cur, p)
try:
sftp.mkdir(cur)
except IOError:
pass
finally:
try:
sftp.close()
except Exception:
pass
await asyncio.to_thread(_do_mkdir)
async def delete(self, root: str, rel: str):
path = _join_remote(root, rel)
def _do_delete():
sftp = self._connect()
try:
# Try file remove first
try:
sftp.remove(path)
return
except IOError:
pass
def _rm_tree(dp: str):
try:
for a in sftp.listdir_attr(dp):
child = _join_remote(dp, a.filename)
if statmod.S_ISDIR(a.st_mode):
_rm_tree(child)
else:
try:
sftp.remove(child)
except Exception:
pass
sftp.rmdir(dp)
except IOError:
pass
_rm_tree(path)
finally:
try:
sftp.close()
except Exception:
pass
await asyncio.to_thread(_do_delete)
async def move(self, root: str, src_rel: str, dst_rel: str):
src = _join_remote(root, src_rel)
dst = _join_remote(root, dst_rel)
def _do_move():
sftp = self._connect()
try:
# ensure dst parent exists
parent = "/" if "/" not in dst.strip("/") else dst.rsplit("/", 1)[0]
parts = [p for p in parent.strip("/").split("/") if p]
cur = "/"
for p in parts:
cur = _join_remote(cur, p)
try:
sftp.mkdir(cur)
except IOError:
pass
sftp.rename(src, dst)
finally:
try:
sftp.close()
except Exception:
pass
await asyncio.to_thread(_do_move)
async def rename(self, root: str, src_rel: str, dst_rel: str):
await self.move(root, src_rel, dst_rel)
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
src = _join_remote(root, src_rel)
dst = _join_remote(root, dst_rel)
def _is_dir() -> bool:
sftp = self._connect()
try:
st = sftp.stat(src)
return statmod.S_ISDIR(st.st_mode)
finally:
try:
sftp.close()
except Exception:
pass
if await asyncio.to_thread(_is_dir):
await self.mkdir(root, dst_rel)
children, _ = await self.list_dir(root, src_rel, page_num=1, page_size=10_000)
for ent in children:
child_src = f"{src_rel.rstrip('/')}/{ent['name']}"
child_dst = f"{dst_rel.rstrip('/')}/{ent['name']}"
await self.copy(root, child_src, child_dst, overwrite)
return
# file copy
data = await self.read_file(root, src_rel)
if not overwrite:
try:
await self.stat_file(root, dst_rel)
raise FileExistsError(dst_rel)
except FileNotFoundError:
pass
await self.write_file(root, dst_rel, data)
async def stat_file(self, root: str, rel: str):
path = _join_remote(root, rel)
def _do_stat():
sftp = self._connect()
try:
st = sftp.stat(path)
is_dir = statmod.S_ISDIR(st.st_mode)
info = {
"name": rel.split("/")[-1],
"is_dir": is_dir,
"size": 0 if is_dir else int(st.st_size or 0),
"mtime": int(st.st_mtime or 0),
"type": "dir" if is_dir else "file",
"path": path,
}
return info
except FileNotFoundError:
raise
except IOError as e:
if getattr(e, "errno", None) == 2:
raise FileNotFoundError(rel)
raise
finally:
try:
sftp.close()
except Exception:
pass
return await asyncio.to_thread(_do_stat)
async def exists(self, root: str, rel: str) -> bool:
try:
await self.stat_file(root, rel)
return True
except FileNotFoundError:
return False
except Exception:
return False
async def stream_file(self, root: str, rel: str, range_header: str | None):
path = _join_remote(root, rel)
def _get_stat():
sftp = self._connect()
try:
st = sftp.stat(path)
return int(st.st_size or 0)
finally:
try:
sftp.close()
except Exception:
pass
file_size = await asyncio.to_thread(_get_stat)
if file_size is None:
raise HTTPException(404, detail="File not found")
mime, _ = mimetypes.guess_type(rel)
content_type = mime or "application/octet-stream"
start = 0
end = file_size - 1
status = 200
headers = {
"Accept-Ranges": "bytes",
"Content-Type": content_type,
"Content-Length": str(file_size),
}
if range_header and range_header.startswith("bytes="):
try:
s, e = (range_header.removeprefix("bytes=").split("-", 1))
if s.strip():
start = int(s)
if e.strip():
end = int(e)
if start >= file_size:
raise HTTPException(416, detail="Requested Range Not Satisfiable")
if end >= file_size:
end = file_size - 1
status = 206
headers["Content-Length"] = str(end - start + 1)
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
except ValueError:
raise HTTPException(400, detail="Invalid Range header")
queue: asyncio.Queue[Optional[bytes]] = asyncio.Queue(maxsize=8)
def _worker():
sftp = self._connect()
try:
with sftp.open(path, "rb") as f:
f.seek(start)
remaining = end - start + 1
chunk_size = 64 * 1024
while remaining > 0:
to_read = chunk_size if remaining > chunk_size else remaining
data = f.read(to_read)
if not data:
break
try:
queue.put_nowait(data)
except Exception:
break
remaining -= len(data)
try:
queue.put_nowait(None)
except Exception:
pass
finally:
try:
sftp.close()
except Exception:
pass
async def agen():
worker_fut = asyncio.to_thread(_worker)
try:
while True:
chunk = await queue.get()
if chunk is None:
break
yield chunk
finally:
try:
await worker_fut
except Exception:
pass
return StreamingResponse(agen(), status_code=status, headers=headers, media_type=content_type)
ADAPTER_TYPE = "sftp"
CONFIG_SCHEMA = [
{"key": "host", "label": "主机", "type": "string", "required": True, "placeholder": "sftp.example.com"},
{"key": "port", "label": "端口", "type": "number", "required": False, "default": 22},
{"key": "username", "label": "用户名", "type": "string", "required": True},
{"key": "password", "label": "密码", "type": "password", "required": True},
{"key": "root", "label": "根路径", "type": "string", "required": True, "placeholder": "/data"},
{"key": "timeout", "label": "超时(秒)", "type": "number", "required": False, "default": 15},
{"key": "allow_unknown_host", "label": "允许未知主机指纹", "type": "boolean", "required": False, "default": True},
]
def ADAPTER_FACTORY(rec: StorageAdapter):
return SFTPAdapter(rec)
+360
View File
@@ -0,0 +1,360 @@
from __future__ import annotations
from typing import List, Dict, Tuple, AsyncIterator
import io
import os
from models import StorageAdapter
from telethon import TelegramClient
from telethon.sessions import StringSession
import socks
# 适配器类型标识
ADAPTER_TYPE = "telegram"
# 适配器配置项定义
CONFIG_SCHEMA = [
{"key": "api_id", "label": "API ID", "type": "string", "required": True, "help_text": "从 my.telegram.org 获取"},
{"key": "api_hash", "label": "API Hash", "type": "password", "required": True, "help_text": "从 my.telegram.org 获取"},
{"key": "session_string", "label": "Session String", "type": "password", "required": True, "help_text": "通过 generate_session.py 生成"},
{"key": "chat_id", "label": "Chat ID", "type": "string", "required": True, "placeholder": "频道/群组的ID或用户名, 例如: -100123456789 或 'channel_username'"},
{"key": "proxy_protocol", "label": "代理协议", "type": "string", "required": False, "placeholder": "例如: socks5, http"},
{"key": "proxy_host", "label": "代理主机", "type": "string", "required": False, "placeholder": "例如: 127.0.0.1"},
{"key": "proxy_port", "label": "代理端口", "type": "number", "required": False, "placeholder": "例如: 1080"},
]
class TelegramAdapter:
"""Telegram 存储适配器 (使用用户 Session)"""
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config
self.api_id = int(cfg.get("api_id"))
self.api_hash = cfg.get("api_hash")
self.session_string = cfg.get("session_string")
self.chat_id_str = cfg.get("chat_id")
# 代理设置
self.proxy_protocol = cfg.get("proxy_protocol")
self.proxy_host = cfg.get("proxy_host")
self.proxy_port = cfg.get("proxy_port")
self.proxy = None
if self.proxy_protocol and self.proxy_host and self.proxy_port:
proto_map = {
"socks5": socks.SOCKS5,
"http": socks.HTTP,
}
proxy_type = proto_map.get(self.proxy_protocol.lower())
if proxy_type:
self.proxy = (proxy_type, self.proxy_host, int(self.proxy_port))
try:
self.chat_id = int(self.chat_id_str)
except (ValueError, TypeError):
self.chat_id = self.chat_id_str
if not all([self.api_id, self.api_hash, self.session_string, self.chat_id]):
raise ValueError("Telegram 适配器需要 api_id, api_hash, session_string 和 chat_id")
def _get_client(self) -> TelegramClient:
"""创建一个新的 TelegramClient 实例"""
return TelegramClient(StringSession(self.session_string), self.api_id, self.api_hash, proxy=self.proxy)
def get_effective_root(self, sub_path: str | None) -> str:
return ""
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
if rel:
return [], 0
client = self._get_client()
entries = []
try:
await client.connect()
messages = await client.get_messages(self.chat_id, limit=200)
for message in messages:
if not message:
continue
media = message.document or message.video or message.photo
if not media:
continue
file_meta = message.file
if not file_meta:
continue
filename = file_meta.name
if not filename:
if message.text and '.' in message.text and len(message.text) < 256 and '\n' not in message.text:
filename = message.text
else:
filename = f"unknown_{message.id}"
size = file_meta.size
if size is None:
# 兼容缺失 size 的情况
if hasattr(media, "size") and media.size is not None:
size = media.size
elif message.photo and getattr(message.photo, "sizes", None):
photo_size = message.photo.sizes[-1]
size = getattr(photo_size, "size", 0) or 0
else:
size = 0
entries.append({
"name": f"{message.id}_{filename}",
"is_dir": False,
"size": size,
"mtime": int(message.date.timestamp()),
"type": "file",
})
finally:
if client.is_connected():
await client.disconnect()
# 排序
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
key = (not item["is_dir"],)
sort_field = sort_by.lower()
if sort_field == "name":
key += (item["name"].lower(),)
elif sort_field == "size":
key += (item["size"],)
elif sort_field == "mtime":
key += (item["mtime"],)
else:
key += (item["name"].lower(),)
return key
entries.sort(key=get_sort_key, reverse=reverse)
total_count = len(entries)
# 分页
start_idx = (page_num - 1) * page_size
end_idx = start_idx + page_size
page_entries = entries[start_idx:end_idx]
return page_entries, total_count
async def read_file(self, root: str, rel: str) -> bytes:
try:
message_id_str, _ = rel.split('_', 1)
message_id = int(message_id_str)
except (ValueError, IndexError):
raise FileNotFoundError(f"无效的文件路径格式: {rel}")
client = self._get_client()
try:
await client.connect()
message = await client.get_messages(self.chat_id, ids=message_id)
if not message or not (message.document or message.video or message.photo):
raise FileNotFoundError(f"在频道 {self.chat_id} 中未找到消息ID为 {message_id} 的文件")
file_bytes = await client.download_media(message, file=bytes)
return file_bytes
finally:
if client.is_connected():
await client.disconnect()
async def write_file(self, root: str, rel: str, data: bytes):
"""将字节数据作为文件上传"""
client = self._get_client()
file_like = io.BytesIO(data)
file_like.name = os.path.basename(rel) or "file"
try:
await client.connect()
await client.send_file(self.chat_id, file_like, caption=file_like.name)
finally:
if client.is_connected():
await client.disconnect()
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
"""以流式方式上传文件"""
client = self._get_client()
filename = os.path.basename(rel) or "file"
import tempfile
temp_dir = tempfile.gettempdir()
temp_path = os.path.join(temp_dir, filename)
total_size = 0
try:
with open(temp_path, "wb") as f:
async for chunk in data_iter:
if chunk:
f.write(chunk)
total_size += len(chunk)
await client.connect()
await client.send_file(self.chat_id, temp_path, caption=filename)
finally:
if os.path.exists(temp_path):
os.remove(temp_path)
if client.is_connected():
await client.disconnect()
return total_size
async def mkdir(self, root: str, rel: str):
raise NotImplementedError("Telegram 适配器不支持创建目录。")
async def delete(self, root: str, rel: str):
"""删除一个文件 (即一条消息)"""
try:
message_id_str, _ = rel.split('_', 1)
message_id = int(message_id_str)
except (ValueError, IndexError):
raise FileNotFoundError(f"无效的文件路径格式,无法解析消息ID: {rel}")
client = self._get_client()
try:
await client.connect()
result = await client.delete_messages(self.chat_id, [message_id])
if not result or not result[0].pts:
raise FileNotFoundError(f"{self.chat_id} 中删除消息 {message_id} 失败,可能消息不存在或无权限")
finally:
if client.is_connected():
await client.disconnect()
async def move(self, root: str, src_rel: str, dst_rel: str):
raise NotImplementedError("Telegram 适配器不支持移动。")
async def rename(self, root: str, src_rel: str, dst_rel: str):
raise NotImplementedError("Telegram 适配器不支持重命名。")
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
raise NotImplementedError("Telegram 适配器不支持复制。")
async def stream_file(self, root: str, rel: str, range_header: str | None):
from fastapi.responses import StreamingResponse
from fastapi import HTTPException
try:
message_id_str, _ = rel.split('_', 1)
message_id = int(message_id_str)
except (ValueError, IndexError):
raise HTTPException(status_code=400, detail=f"无效的文件路径格式: {rel}")
client = self._get_client()
try:
await client.connect()
message = await client.get_messages(self.chat_id, ids=message_id)
media = message.document or message.video or message.photo
if not message or not media:
raise FileNotFoundError(f"在频道 {self.chat_id} 中未找到消息ID为 {message_id} 的文件")
file_meta = message.file
file_size = file_meta.size if file_meta and file_meta.size is not None else None
if file_size is None:
if hasattr(media, "size") and media.size is not None:
file_size = media.size
elif message.photo and getattr(message.photo, "sizes", None):
photo_size = message.photo.sizes[-1]
file_size = getattr(photo_size, "size", 0) or 0
else:
file_size = 0
mime_type = None
if file_meta and getattr(file_meta, "mime_type", None):
mime_type = file_meta.mime_type
if not mime_type:
if hasattr(media, "mime_type") and media.mime_type:
mime_type = media.mime_type
elif message.photo:
mime_type = "image/jpeg"
else:
mime_type = "application/octet-stream"
start = 0
end = file_size - 1
status = 200
headers = {
"Accept-Ranges": "bytes",
"Content-Type": mime_type,
"Content-Length": str(file_size),
}
if range_header:
try:
range_val = range_header.strip().partition("=")[2]
s, _, e = range_val.partition("-")
start = int(s) if s else 0
end = int(e) if e else file_size - 1
if start >= file_size or end >= file_size or start > end:
raise HTTPException(status_code=416, detail="Requested Range Not Satisfiable")
status = 206
headers["Content-Length"] = str(end - start + 1)
headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
except ValueError:
raise HTTPException(status_code=400, detail="Invalid Range header")
async def iterator():
try:
limit = end - start + 1
downloaded = 0
async for chunk in client.iter_download(media, offset=start):
if downloaded + len(chunk) > limit:
yield chunk[:limit - downloaded]
break
yield chunk
downloaded += len(chunk)
if downloaded >= limit:
break
finally:
if client.is_connected():
await client.disconnect()
return StreamingResponse(iterator(), status_code=status, headers=headers)
except FileNotFoundError as e:
if client.is_connected():
await client.disconnect()
raise HTTPException(status_code=404, detail=str(e))
except Exception as e:
if client.is_connected():
await client.disconnect()
raise HTTPException(status_code=500, detail=f"Streaming failed: {str(e)}")
async def stat_file(self, root: str, rel: str):
try:
message_id_str, filename = rel.split('_', 1)
message_id = int(message_id_str)
except (ValueError, IndexError):
raise FileNotFoundError(f"无效的文件路径格式: {rel}")
client = self._get_client()
try:
await client.connect()
message = await client.get_messages(self.chat_id, ids=message_id)
media = message.document or message.video or message.photo
if not message or not media:
raise FileNotFoundError(f"在频道 {self.chat_id} 中未找到消息ID为 {message_id} 的文件")
file_meta = message.file
size = file_meta.size if file_meta and file_meta.size is not None else None
if size is None:
if hasattr(media, "size") and media.size is not None:
size = media.size
elif message.photo and getattr(message.photo, "sizes", None):
photo_size = message.photo.sizes[-1]
size = getattr(photo_size, "size", 0) or 0
else:
size = 0
return {
"name": rel,
"is_dir": False,
"size": size,
"mtime": int(message.date.timestamp()),
"type": "file",
}
finally:
if client.is_connected():
await client.disconnect()
def ADAPTER_FACTORY(rec: StorageAdapter) -> TelegramAdapter:
return TelegramAdapter(rec)
+493
View File
@@ -0,0 +1,493 @@
from __future__ import annotations
from typing import List, Dict, Optional, Tuple, AsyncIterator
import httpx
from urllib.parse import urljoin, quote
from urllib.parse import urlparse, unquote
import xml.etree.ElementTree as ET
from models import StorageAdapter
import mimetypes
import logging
from fastapi import HTTPException
from fastapi.responses import StreamingResponse, Response
NS = {"d": "DAV:"}
class WebDAVAdapter:
def __init__(self, record: StorageAdapter):
self.record = record
cfg = record.config
self.base_url: str = cfg.get("base_url", "").rstrip('/') + '/'
if not self.base_url.startswith("http"):
raise ValueError("webdav requires base_url http/https")
self.username = cfg.get("username")
self.password = cfg.get("password")
self.timeout = cfg.get("timeout", 15)
def get_effective_root(self, sub_path: str | None) -> str:
base_url = self.record.config.get("base_url", "").rstrip('/') + '/'
if sub_path:
return base_url + sub_path.strip('/') + '/'
return base_url
def _client(self):
auth = (self.username, self.password) if self.username else None
return httpx.AsyncClient(auth=auth, timeout=self.timeout, follow_redirects=True)
def _build_url(self, rel: str):
rel = rel.strip('/')
return self.base_url if not rel else urljoin(self.base_url, quote(rel) + ('/' if rel.endswith('/') else ''))
async def list_dir(self, root: str, rel: str, page_num: int = 1, page_size: int = 50, sort_by: str = "name", sort_order: str = "asc") -> Tuple[List[Dict], int]:
raw_url = self._build_url(rel)
url = raw_url if raw_url.endswith('/') else raw_url + '/'
depth = "1"
body = """<?xml version="1.0" encoding="utf-8" ?>
<d:propfind xmlns:d="DAV:">
<d:prop>
<d:displayname />
<d:getcontentlength />
<d:getlastmodified />
<d:resourcetype />
</d:prop>
</d:propfind>"""
async with self._client() as client:
resp = await client.request("PROPFIND", url, data=body, headers={"Depth": depth})
resp.raise_for_status()
xml_text = resp.text
root_el = ET.fromstring(xml_text)
all_entries: List[Dict] = []
parsed_req = urlparse(url)
base_path = parsed_req.path
if not base_path.endswith('/'):
base_path += '/'
seen = set()
for resp_el in root_el.findall("d:response", NS):
href_el = resp_el.find("d:href", NS)
if href_el is None:
continue
href = (href_el.text or "")
parsed_href = urlparse(href)
href_path = parsed_href.path or ""
if not href_path.startswith(base_path):
continue
rel_path = href_path[len(base_path):].strip('/')
if rel_path == "":
continue
name = unquote(rel_path.split('/')[0]).rstrip('/')
if not name or name in seen:
continue
seen.add(name)
propstat = resp_el.find("d:propstat", NS)
if propstat is None:
continue
prop = propstat.find("d:prop", NS)
if prop is None:
continue
size_el = prop.find("d:getcontentlength", NS)
lm_el = prop.find("d:getlastmodified", NS)
rt_el = prop.find("d:resourcetype", NS)
is_dir = rt_el.find(
"d:collection", NS) is not None if rt_el is not None else href_path.endswith('/')
size = int(
size_el.text) if size_el is not None and size_el.text and size_el.text.isdigit() else 0
from email.utils import parsedate_to_datetime
mtime = 0
if lm_el is not None and lm_el.text:
try:
mtime = int(parsedate_to_datetime(lm_el.text).timestamp())
except Exception:
mtime = 0
all_entries.append({
"name": name,
"is_dir": is_dir,
"size": 0 if is_dir else size,
"mtime": mtime,
"type": "dir" if is_dir else "file",
})
# 排序所有条目
reverse = sort_order.lower() == "desc"
def get_sort_key(item):
key = (not item["is_dir"],)
sort_field = sort_by.lower()
if sort_field == "name":
key += (item["name"].lower(),)
elif sort_field == "size":
key += (item["size"],)
elif sort_field == "mtime":
key += (item["mtime"],)
else:
key += (item["name"].lower(),)
return key
all_entries.sort(key=get_sort_key, reverse=reverse)
total_count = len(all_entries)
# 应用分页
start_idx = (page_num - 1) * page_size
end_idx = start_idx + page_size
page_entries = all_entries[start_idx:end_idx]
return page_entries, total_count
async def read_file(self, root: str, rel: str) -> bytes:
url = self._build_url(rel)
async with self._client() as client:
resp = await client.get(url)
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
return resp.content
async def write_file(self, root: str, rel: str, data: bytes):
url = self._build_url(rel)
async with self._client() as client:
resp = await client.put(url, content=data)
resp.raise_for_status()
async def mkdir(self, root: str, rel: str):
url = self._build_url(rel.rstrip('/') + '/')
async with self._client() as client:
resp = await client.request("MKCOL", url)
if resp.status_code not in (201, 405):
resp.raise_for_status()
async def delete(self, root: str, rel: str):
url = self._build_url(rel)
async with self._client() as client:
resp = await client.delete(url)
if resp.status_code not in (204, 200, 404):
resp.raise_for_status()
async def move(self, root: str, src_rel: str, dst_rel: str):
src_url = self._build_url(src_rel)
dst_url = self._build_url(dst_rel)
async with self._client() as client:
resp = await client.request("MOVE", src_url, headers={"Destination": dst_url})
resp.raise_for_status()
async def rename(self, root: str, src_rel: str, dst_rel: str):
src_url = self._build_url(src_rel)
dst_url = self._build_url(dst_rel)
async with self._client() as client:
resp = await client.request("MOVE", src_url, headers={"Destination": dst_url})
resp.raise_for_status()
async def get_file_size(self, root: str, rel: str) -> int:
"""获取文件大小"""
url = self._build_url(rel)
async with self._client() as client:
# 使用HEAD请求获取文件信息
resp = await client.head(url)
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
content_length = resp.headers.get('content-length')
if content_length:
return int(content_length)
# 如果HEAD不返回content-length,尝试PROPFIND
body = """<?xml version="1.0" encoding="utf-8" ?>
<d:propfind xmlns:d="DAV:">
<d:prop>
<d:getcontentlength />
</d:prop>
</d:propfind>"""
resp = await client.request("PROPFIND", url, data=body, headers={"Depth": "0"})
resp.raise_for_status()
root_el = ET.fromstring(resp.text)
for resp_el in root_el.findall("d:response", NS):
propstat = resp_el.find("d:propstat", NS)
if propstat is None:
continue
prop = propstat.find("d:prop", NS)
if prop is None:
continue
size_el = prop.find("d:getcontentlength", NS)
if size_el is not None and size_el.text and size_el.text.isdigit():
return int(size_el.text)
return 0
async def read_file_range(self, root: str, rel: str, start: int, end: Optional[int] = None) -> bytes:
"""读取文件的指定范围"""
url = self._build_url(rel)
# 构建Range头
if end is None:
range_header = f"bytes={start}-"
else:
range_header = f"bytes={start}-{end}"
async with self._client() as client:
resp = await client.get(url, headers={"Range": range_header})
if resp.status_code == 404:
raise FileNotFoundError(rel)
if resp.status_code not in (200, 206): # 206是Partial Content
resp.raise_for_status()
return resp.content
async def stream_file(self, root: str, rel: str, range_header: str | None):
url = self._build_url(rel)
mime, _ = mimetypes.guess_type(rel)
content_type = mime or "application/octet-stream"
logger = logging.getLogger(__name__)
timeout = self.timeout
auth = (self.username, self.password) if self.username else None
client_start = 0
client_end = None
status_code = 200
if range_header and range_header.startswith("bytes="):
status_code = 206
part = range_header.removeprefix("bytes=")
s, e = part.split("-", 1)
if s.strip():
client_start = int(s)
if e.strip():
client_end = int(e)
total_size = None
accept_ranges = False
async with httpx.AsyncClient(timeout=timeout, auth=auth, follow_redirects=True) as client:
try:
head_resp = await client.head(url)
if head_resp.status_code == 404:
raise HTTPException(404, detail="File not found")
if head_resp.status_code == 200:
cl = head_resp.headers.get("Content-Length")
if cl and cl.isdigit():
total_size = int(cl)
ar = head_resp.headers.get("Accept-Ranges", "").lower()
accept_ranges = "bytes" in ar
except HTTPException:
raise
except Exception as e:
logger.debug("HEAD failed %s err=%s", url, e)
if total_size is None and (client_end is None):
try:
probe_req = client.build_request("GET", url, headers={"Range": "bytes=0-0"})
probe_resp = await client.send(probe_req, stream=True)
if probe_resp.status_code in (200, 206):
cr = probe_resp.headers.get("Content-Range")
if cr and "/" in cr:
try:
total_size = int(cr.rsplit("/", 1)[1])
except Exception:
pass
await probe_resp.aclose()
except Exception as e:
logger.debug("Probe 0-0 failed %s err=%s", url, e)
if total_size is not None and client_end is None:
client_end = total_size - 1
if client_end is not None and client_end < client_start:
raise HTTPException(416, detail="Requested Range Not Satisfiable")
# 若客户端未请求范围且上游不支持 Range,直接透传
if status_code == 200 and (range_header is None) and not accept_ranges:
async with httpx.AsyncClient(timeout=timeout, auth=auth, follow_redirects=True) as client:
req = client.build_request("GET", url)
resp = await client.send(req, stream=True)
if resp.status_code == 404:
await resp.aclose()
raise HTTPException(404, detail="File not found")
upstream_ct = resp.headers.get("Content-Type", content_type)
async def passthrough():
try:
async for chunk in resp.aiter_bytes():
if chunk:
yield chunk
finally:
await resp.aclose()
return StreamingResponse(passthrough(), status_code=resp.status_code,
headers={"Accept-Ranges": "bytes",
"X-VFS-Remote-Status": str(resp.status_code)},
media_type=upstream_ct)
SEGMENT_SIZE = 5 * 1024 * 1024
MAX_RETRY_PER_SEG = 3
FIRST_BYTE_MAX_RETRY = 3
resp_headers = {
"Accept-Ranges": "bytes",
"Content-Type": content_type,
"X-VFS-Segmented": "1",
}
if status_code == 206 and total_size is not None:
resp_headers["Content-Range"] = f"bytes {client_start}-{client_end}/{total_size}"
async def segmented_body():
current = client_start
first_byte_sent = False
while True:
if client_end is not None and current > client_end:
break
seg_start = current
seg_end = (min(seg_start + SEGMENT_SIZE - 1, client_end)
if client_end is not None else seg_start + SEGMENT_SIZE - 1)
attempt = 0
ok = False
while attempt < MAX_RETRY_PER_SEG and not ok:
attempt += 1
headers_req = {"Range": f"bytes={seg_start}-{seg_end}"}
try:
async with httpx.AsyncClient(timeout=timeout, auth=auth, follow_redirects=True) as cseg:
req = cseg.build_request("GET", url, headers=headers_req)
rseg = await cseg.send(req, stream=True)
if rseg.status_code in (200, 206):
async for chunk in rseg.aiter_bytes():
if chunk:
first_byte_sent = True
yield chunk
await rseg.aclose()
ok = True
elif rseg.status_code == 404:
await rseg.aclose()
if not first_byte_sent:
raise HTTPException(404, detail="File not found")
return
else:
await rseg.aclose()
logger.warning("Segment unexpected status %s %s-%s %s", rel, seg_start, seg_end, rseg.status_code)
if not ok:
continue
except (httpx.ReadError, httpx.HTTPError, httpx.StreamError) as e:
if not first_byte_sent and attempt >= FIRST_BYTE_MAX_RETRY:
raise HTTPException(502, detail=f"Upstream error before first byte err={e}")
logger.warning("Segment error %s %s-%s attempt=%d err=%s", rel, seg_start, seg_end, attempt, e)
except Exception as e:
if not first_byte_sent:
raise
logger.error("Segment unexpected %s %s-%s attempt=%d err=%s", rel, seg_start, seg_end, attempt, e)
if not ok:
logger.error("Abort streaming %s at %s-%s", rel, seg_start, seg_end)
break
current = seg_end + 1
if client_end is None:
continue
if current > client_end:
break
return StreamingResponse(segmented_body(), status_code=status_code, headers=resp_headers, media_type=content_type)
async def stat_file(self, root: str, rel: str):
url = self._build_url(rel)
async with self._client() as client:
# PROPFIND 获取属性
body = """<?xml version="1.0" encoding="utf-8" ?>
<d:propfind xmlns:d="DAV:">
<d:prop>
<d:getcontentlength />
<d:getlastmodified />
<d:resourcetype />
</d:prop>
</d:propfind>"""
resp = await client.request("PROPFIND", url, data=body, headers={"Depth": "0"})
if resp.status_code == 404:
raise FileNotFoundError(rel)
resp.raise_for_status()
root_el = ET.fromstring(resp.text)
info = {
"name": rel.split("/")[-1],
"is_dir": False,
"size": None,
"mtime": None,
"type": "file",
"path": url,
}
for resp_el in root_el.findall("d:response", NS):
propstat = resp_el.find("d:propstat", NS)
if propstat is None:
continue
prop = propstat.find("d:prop", NS)
if prop is None:
continue
size_el = prop.find("d:getcontentlength", NS)
lm_el = prop.find("d:getlastmodified", NS)
rt_el = prop.find("d:resourcetype", NS)
is_dir = rt_el.find("d:collection", NS) is not None if rt_el is not None else False
info["is_dir"] = is_dir
info["type"] = "dir" if is_dir else "file"
if size_el is not None and size_el.text and size_el.text.isdigit():
info["size"] = int(size_el.text)
elif info["size"] is None:
info["size"] = 0
if lm_el is not None and lm_el.text:
from email.utils import parsedate_to_datetime
try:
info["mtime"] = int(parsedate_to_datetime(lm_el.text).timestamp())
except Exception:
info["mtime"] = 0
elif info["mtime"] is None:
info["mtime"] = 0
# exif信息
exif = None
if not info["is_dir"]:
mime, _ = mimetypes.guess_type(info["name"])
if mime and mime.startswith("image/"):
try:
resp_img = await client.get(url)
if resp_img.status_code == 200:
from PIL import Image
from io import BytesIO
img = Image.open(BytesIO(resp_img.content))
exif_data = img._getexif()
if exif_data:
exif = {str(k): str(v) for k, v in exif_data.items()}
except Exception:
exif = None
info["exif"] = exif
return info
async def exists(self, root: str, rel: str) -> bool:
url = self._build_url(rel)
async with self._client() as client:
try:
r = await client.head(url)
return r.status_code in (200, 204)
except Exception:
return False
async def write_file_stream(self, root: str, rel: str, data_iter: AsyncIterator[bytes]):
url = self._build_url(rel)
async def agen():
async for chunk in data_iter:
if chunk:
yield chunk
async with self._client() as client:
resp = await client.put(url, content=agen())
resp.raise_for_status()
return True
async def copy(self, root: str, src_rel: str, dst_rel: str, overwrite: bool = False):
src_url = self._build_url(src_rel)
dst_url = self._build_url(dst_rel)
headers = {
"Destination": dst_url,
"Overwrite": "T" if overwrite else "F"
}
async with self._client() as client:
resp = await client.request("COPY", src_url, headers=headers)
if resp.status_code == 412:
raise FileExistsError(dst_rel)
if resp.status_code == 404:
raise FileNotFoundError(src_rel)
resp.raise_for_status()
ADAPTER_TYPE = "webdav"
CONFIG_SCHEMA = [
{"key": "base_url", "label": "基础地址", "type": "string",
"required": True, "placeholder": "https://example.com/dav/"},
{"key": "username", "label": "用户名", "type": "string", "required": False},
{"key": "password", "label": "密码", "type": "password", "required": False},
{"key": "timeout",
"label": "超时(秒)", "type": "number", "required": False, "default": 15},
]
def ADAPTER_FACTORY(rec): return WebDAVAdapter(rec)