mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-05 23:47:41 +08:00
Merge branch 'jxxghp:v2' into v2
This commit is contained in:
@@ -339,7 +339,8 @@ class DownloadChain(ChainBase):
|
|||||||
meta=_meta,
|
meta=_meta,
|
||||||
mediainfo=_media,
|
mediainfo=_media,
|
||||||
torrentinfo=_torrent,
|
torrentinfo=_torrent,
|
||||||
download_episodes=download_episodes
|
download_episodes=download_episodes,
|
||||||
|
username=username,
|
||||||
)
|
)
|
||||||
# 下载成功后处理
|
# 下载成功后处理
|
||||||
self.download_added(context=context, download_dir=download_dir, torrent_path=torrent_file)
|
self.download_added(context=context, download_dir=download_dir, torrent_path=torrent_file)
|
||||||
|
|||||||
@@ -241,6 +241,7 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
|
|||||||
link=link,
|
link=link,
|
||||||
username=username
|
username=username
|
||||||
),
|
),
|
||||||
|
meta=metainfo,
|
||||||
mediainfo=mediainfo,
|
mediainfo=mediainfo,
|
||||||
username=username
|
username=username
|
||||||
)
|
)
|
||||||
@@ -1023,7 +1024,8 @@ class SubscribeChain(ChainBase, metaclass=Singleton):
|
|||||||
),
|
),
|
||||||
meta=meta,
|
meta=meta,
|
||||||
mediainfo=mediainfo,
|
mediainfo=mediainfo,
|
||||||
msgstr=msgstr
|
msgstr=msgstr,
|
||||||
|
username=subscribe.username
|
||||||
)
|
)
|
||||||
# 发送事件
|
# 发送事件
|
||||||
EventManager().send_event(EventType.SubscribeComplete, {
|
EventManager().send_event(EventType.SubscribeComplete, {
|
||||||
|
|||||||
@@ -1385,5 +1385,6 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
|||||||
meta=meta,
|
meta=meta,
|
||||||
mediainfo=mediainfo,
|
mediainfo=mediainfo,
|
||||||
transferinfo=transferinfo,
|
transferinfo=transferinfo,
|
||||||
season_episode=season_episode
|
season_episode=season_episode,
|
||||||
|
username=username
|
||||||
)
|
)
|
||||||
|
|||||||
+9
-5
@@ -465,13 +465,15 @@ class PluginManager(metaclass=Singleton):
|
|||||||
}]
|
}]
|
||||||
"""
|
"""
|
||||||
ret_apis = []
|
ret_apis = []
|
||||||
for plugin_id, plugin in self._running_plugins.items():
|
if pid:
|
||||||
|
plugins = {pid: self._running_plugins.get(pid)}
|
||||||
|
else:
|
||||||
|
plugins = self._running_plugins
|
||||||
|
for plugin_id, plugin in plugins.items():
|
||||||
if pid and pid != plugin_id:
|
if pid and pid != plugin_id:
|
||||||
continue
|
continue
|
||||||
if hasattr(plugin, "get_api") and ObjectUtils.check_method(plugin.get_api):
|
if hasattr(plugin, "get_api") and ObjectUtils.check_method(plugin.get_api):
|
||||||
try:
|
try:
|
||||||
if not plugin.get_state():
|
|
||||||
continue
|
|
||||||
apis = plugin.get_api() or []
|
apis = plugin.get_api() or []
|
||||||
for api in apis:
|
for api in apis:
|
||||||
api["path"] = f"/{plugin_id}{api['path']}"
|
api["path"] = f"/{plugin_id}{api['path']}"
|
||||||
@@ -831,7 +833,8 @@ class PluginManager(metaclass=Singleton):
|
|||||||
logger.debug(f"获取插件是否在本地包中存在失败,{e}")
|
logger.debug(f"获取插件是否在本地包中存在失败,{e}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def get_plugins_from_market(self, market: str, package_version: Optional[str] = None) -> Optional[List[schemas.Plugin]]:
|
def get_plugins_from_market(self, market: str, package_version: Optional[str] = None) -> Optional[
|
||||||
|
List[schemas.Plugin]]:
|
||||||
"""
|
"""
|
||||||
从指定的市场获取插件信息
|
从指定的市场获取插件信息
|
||||||
:param market: 市场的 URL 或标识
|
:param market: 市场的 URL 或标识
|
||||||
@@ -845,7 +848,8 @@ class PluginManager(metaclass=Singleton):
|
|||||||
# 获取在线插件
|
# 获取在线插件
|
||||||
online_plugins = self.pluginhelper.get_plugins(market, package_version)
|
online_plugins = self.pluginhelper.get_plugins(market, package_version)
|
||||||
if online_plugins is None:
|
if online_plugins is None:
|
||||||
logger.warning(f"获取{package_version if package_version else ''}插件库失败:{market},请检查 GitHub 网络连接")
|
logger.warning(
|
||||||
|
f"获取{package_version if package_version else ''}插件库失败:{market},请检查 GitHub 网络连接")
|
||||||
return []
|
return []
|
||||||
ret_plugins = []
|
ret_plugins = []
|
||||||
add_time = len(online_plugins)
|
add_time = len(online_plugins)
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ class TemplateContextBuilder:
|
|||||||
transferinfo: Optional[TransferInfo] = None,
|
transferinfo: Optional[TransferInfo] = None,
|
||||||
file_extension: Optional[str] = None,
|
file_extension: Optional[str] = None,
|
||||||
episodes_info: Optional[List[TmdbEpisode]] = None,
|
episodes_info: Optional[List[TmdbEpisode]] = None,
|
||||||
include_raw_objects: bool = False,
|
include_raw_objects: bool = True,
|
||||||
**kwargs
|
**kwargs
|
||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
@@ -80,8 +80,11 @@ class TemplateContextBuilder:
|
|||||||
"en_title": self.__convert_invalid_characters(mediainfo.en_title),
|
"en_title": self.__convert_invalid_characters(mediainfo.en_title),
|
||||||
# 原语种标题
|
# 原语种标题
|
||||||
"original_title": self.__convert_invalid_characters(mediainfo.original_title),
|
"original_title": self.__convert_invalid_characters(mediainfo.original_title),
|
||||||
|
# 季号
|
||||||
|
"season": self._context.get("season") or mediainfo.season,
|
||||||
# 年份
|
# 年份
|
||||||
"year": mediainfo.year or self._context.get("year"),
|
"year": mediainfo.year or self._context.get("year"),
|
||||||
|
# 媒体标题 + 年份
|
||||||
"title_year": mediainfo.title_year or self._context.get("title_year"),
|
"title_year": mediainfo.title_year or self._context.get("title_year"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -905,8 +905,7 @@ class FileManagerModule(_ModuleBase):
|
|||||||
def __transfer_file(self, fileitem: FileItem, mediainfo: MediaInfo,
|
def __transfer_file(self, fileitem: FileItem, mediainfo: MediaInfo,
|
||||||
source_oper: StorageBase, target_oper: StorageBase,
|
source_oper: StorageBase, target_oper: StorageBase,
|
||||||
target_storage: str, target_file: Path,
|
target_storage: str, target_file: Path,
|
||||||
transfer_type: str, over_flag: Optional[bool] = False) -> Tuple[
|
transfer_type: str, over_flag: Optional[bool] = False) -> Tuple[Optional[FileItem], str]:
|
||||||
Optional[FileItem], str]:
|
|
||||||
"""
|
"""
|
||||||
整理一个文件,同时处理其他相关文件
|
整理一个文件,同时处理其他相关文件
|
||||||
:param fileitem: 原文件
|
:param fileitem: 原文件
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
import base64
|
import base64
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import io
|
||||||
import secrets
|
import secrets
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import List, Dict, Optional, Tuple, Union
|
from typing import List, Dict, Optional, Tuple, Union
|
||||||
|
|
||||||
@@ -24,6 +26,10 @@ class NoCheckInException(Exception):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class SessionInvalidException(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
class AliPan(StorageBase, metaclass=Singleton):
|
class AliPan(StorageBase, metaclass=Singleton):
|
||||||
"""
|
"""
|
||||||
阿里云盘相关操作
|
阿里云盘相关操作
|
||||||
@@ -50,6 +56,13 @@ class AliPan(StorageBase, metaclass=Singleton):
|
|||||||
# CID和路径缓存
|
# CID和路径缓存
|
||||||
_id_cache: Dict[str, Tuple[str, str]] = {}
|
_id_cache: Dict[str, Tuple[str, str]] = {}
|
||||||
|
|
||||||
|
# 最大线程数
|
||||||
|
MAX_WORKERS = 10
|
||||||
|
# 最大分片大小(1GB)
|
||||||
|
MAX_PART_SIZE = 1024 * 1024 * 1024
|
||||||
|
# 最小分片大小(100MB)
|
||||||
|
MIN_PART_SIZE = 100 * 1024 * 1024
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.session = requests.Session()
|
self.session = requests.Session()
|
||||||
@@ -177,7 +190,7 @@ class AliPan(StorageBase, metaclass=Singleton):
|
|||||||
确认登录后,获取相关token
|
确认登录后,获取相关token
|
||||||
"""
|
"""
|
||||||
if not self._auth_state:
|
if not self._auth_state:
|
||||||
raise Exception("【阿里云盘】请先生成二维码")
|
raise SessionInvalidException("【阿里云盘】请先生成二维码")
|
||||||
resp = self.session.post(
|
resp = self.session.post(
|
||||||
f"{self.base_url}/oauth/access_token",
|
f"{self.base_url}/oauth/access_token",
|
||||||
json={
|
json={
|
||||||
@@ -188,7 +201,7 @@ class AliPan(StorageBase, metaclass=Singleton):
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
if resp is None:
|
if resp is None:
|
||||||
raise Exception("【阿里云盘】获取 access_token 失败")
|
raise SessionInvalidException("【阿里云盘】获取 access_token 失败")
|
||||||
result = resp.json()
|
result = resp.json()
|
||||||
if result.get("code"):
|
if result.get("code"):
|
||||||
raise Exception(f"【阿里云盘】{result.get('code')} - {result.get('message')}!")
|
raise Exception(f"【阿里云盘】{result.get('code')} - {result.get('message')}!")
|
||||||
@@ -199,7 +212,7 @@ class AliPan(StorageBase, metaclass=Singleton):
|
|||||||
刷新access_token
|
刷新access_token
|
||||||
"""
|
"""
|
||||||
if not refresh_token:
|
if not refresh_token:
|
||||||
raise Exception("【阿里云盘】会话失效,请重新扫码登录!")
|
raise SessionInvalidException("【阿里云盘】会话失效,请重新扫码登录!")
|
||||||
resp = self.session.post(
|
resp = self.session.post(
|
||||||
f"{self.base_url}/oauth/access_token",
|
f"{self.base_url}/oauth/access_token",
|
||||||
json={
|
json={
|
||||||
@@ -335,6 +348,8 @@ class AliPan(StorageBase, metaclass=Singleton):
|
|||||||
"""
|
"""
|
||||||
if not fileinfo:
|
if not fileinfo:
|
||||||
return schemas.FileItem()
|
return schemas.FileItem()
|
||||||
|
if not parent.endswith("/"):
|
||||||
|
parent += "/"
|
||||||
if fileinfo.get("type") == "folder":
|
if fileinfo.get("type") == "folder":
|
||||||
return schemas.FileItem(
|
return schemas.FileItem(
|
||||||
storage=self.schema.value,
|
storage=self.schema.value,
|
||||||
@@ -437,7 +452,7 @@ class AliPan(StorageBase, metaclass=Singleton):
|
|||||||
"/adrive/v1.0/openFile/create",
|
"/adrive/v1.0/openFile/create",
|
||||||
json={
|
json={
|
||||||
"drive_id": parent_item.drive_id,
|
"drive_id": parent_item.drive_id,
|
||||||
"parent_file_id": parent_item.fileid,
|
"parent_file_id": parent_item.fileid or "root",
|
||||||
"name": name,
|
"name": name,
|
||||||
"type": "folder"
|
"type": "folder"
|
||||||
}
|
}
|
||||||
@@ -583,13 +598,6 @@ class AliPan(StorageBase, metaclass=Singleton):
|
|||||||
raise Exception(resp.get("message"))
|
raise Exception(resp.get("message"))
|
||||||
return resp.get('part_info_list', [])
|
return resp.get('part_info_list', [])
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _upload_part(upload_url: str, data: bytes):
|
|
||||||
"""
|
|
||||||
上传单个分片
|
|
||||||
"""
|
|
||||||
return requests.put(upload_url, data=data)
|
|
||||||
|
|
||||||
def _list_uploaded_parts(self, drive_id: str, file_id: str, upload_id: str) -> dict:
|
def _list_uploaded_parts(self, drive_id: str, file_id: str, upload_id: str) -> dict:
|
||||||
"""
|
"""
|
||||||
获取已上传分片列表
|
获取已上传分片列表
|
||||||
@@ -628,22 +636,77 @@ class AliPan(StorageBase, metaclass=Singleton):
|
|||||||
raise Exception(resp.get("message"))
|
raise Exception(resp.get("message"))
|
||||||
return resp
|
return resp
|
||||||
|
|
||||||
|
def _calc_parts(self, file_size: int) -> Tuple[int, int]:
|
||||||
|
"""
|
||||||
|
计算最优分片大小和线程数,在最大分片大小和最小分片大小之间取最优值
|
||||||
|
:param file_size: 文件大小
|
||||||
|
:return: 分片大小,线程数
|
||||||
|
"""
|
||||||
|
if file_size <= self.MIN_PART_SIZE:
|
||||||
|
return file_size, 1
|
||||||
|
if file_size >= self.MAX_PART_SIZE:
|
||||||
|
part_size = self.MAX_PART_SIZE
|
||||||
|
else:
|
||||||
|
part_size = max(self.MIN_PART_SIZE, file_size // self.MAX_WORKERS)
|
||||||
|
return part_size, (file_size + part_size - 1) // part_size
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _log_progress(desc: str, total: int) -> tqdm:
|
||||||
|
"""
|
||||||
|
创建一个可以输出到日志的进度条
|
||||||
|
"""
|
||||||
|
|
||||||
|
class TqdmToLogger(io.StringIO):
|
||||||
|
def write(s, buf): # noqa
|
||||||
|
buf = buf.strip('\r\n\t ')
|
||||||
|
if buf:
|
||||||
|
logger.info(buf)
|
||||||
|
|
||||||
|
return tqdm(
|
||||||
|
total=total,
|
||||||
|
unit='B',
|
||||||
|
unit_scale=True,
|
||||||
|
desc=desc,
|
||||||
|
file=TqdmToLogger(),
|
||||||
|
mininterval=1.0,
|
||||||
|
maxinterval=5.0,
|
||||||
|
miniters=1
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _upload_part(upload_url: str, data: bytes, part_num: int) -> Tuple[int, str, int]:
|
||||||
|
"""
|
||||||
|
上传单个分片
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
response = requests.put(upload_url, data=data)
|
||||||
|
if response and response.status_code == 200:
|
||||||
|
logger.info(f"【阿里云盘】分片 {part_num} 上传完成")
|
||||||
|
return part_num, response.headers.get('ETag', ''), len(data)
|
||||||
|
else:
|
||||||
|
raise Exception(f"上传失败: {response.status_code if response else 'No Response'}")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"【阿里云盘】分片 {part_num} 上传失败: {str(e)}")
|
||||||
|
raise
|
||||||
|
|
||||||
def upload(self, target_dir: schemas.FileItem, local_path: Path,
|
def upload(self, target_dir: schemas.FileItem, local_path: Path,
|
||||||
new_name: Optional[str] = None) -> Optional[schemas.FileItem]:
|
new_name: Optional[str] = None) -> Optional[schemas.FileItem]:
|
||||||
"""
|
"""
|
||||||
文件上传:分片、支持秒传
|
文件上传:多线程分片、支持秒传
|
||||||
"""
|
"""
|
||||||
target_name = new_name or local_path.name
|
target_name = new_name or local_path.name
|
||||||
target_path = Path(target_dir.path) / target_name
|
target_path = Path(target_dir.path) / target_name
|
||||||
file_size = local_path.stat().st_size
|
file_size = local_path.stat().st_size
|
||||||
|
|
||||||
# 1. 创建文件并检查秒传
|
# 1. 计算分片大小和线程数
|
||||||
chunk_size = 100 * 1024 * 1024 # 分片大小 100M
|
part_size, workers = self._calc_parts(file_size)
|
||||||
|
|
||||||
|
# 2. 创建文件并检查秒传
|
||||||
create_res = self._create_file(drive_id=target_dir.drive_id,
|
create_res = self._create_file(drive_id=target_dir.drive_id,
|
||||||
parent_file_id=target_dir.fileid,
|
parent_file_id=target_dir.fileid,
|
||||||
file_name=target_name,
|
file_name=target_name,
|
||||||
file_path=local_path,
|
file_path=local_path,
|
||||||
chunk_size=chunk_size)
|
chunk_size=part_size)
|
||||||
if create_res.get('rapid_upload', False):
|
if create_res.get('rapid_upload', False):
|
||||||
logger.info(f"【阿里云盘】{target_name} 秒传完成!")
|
logger.info(f"【阿里云盘】{target_name} 秒传完成!")
|
||||||
return self.get_item(target_path)
|
return self.get_item(target_path)
|
||||||
@@ -652,93 +715,78 @@ class AliPan(StorageBase, metaclass=Singleton):
|
|||||||
logger.info(f"【阿里云盘】{target_name} 已存在")
|
logger.info(f"【阿里云盘】{target_name} 已存在")
|
||||||
return self.get_item(target_path)
|
return self.get_item(target_path)
|
||||||
|
|
||||||
# 2. 准备分片上传参数
|
# 3. 准备分片上传参数
|
||||||
file_id = create_res.get('file_id')
|
file_id = create_res.get('file_id')
|
||||||
if not file_id:
|
if not file_id:
|
||||||
logger.warn(f"【阿里云盘】创建 {target_name} 文件失败!")
|
logger.warn(f"【阿里云盘】创建 {target_name} 文件失败!")
|
||||||
return None
|
return None
|
||||||
upload_id = create_res.get('upload_id')
|
upload_id = create_res.get('upload_id')
|
||||||
part_info_list = create_res.get('part_info_list')
|
part_info_list = create_res.get('part_info_list')
|
||||||
uploaded_parts = set()
|
uploaded_parts = {}
|
||||||
|
|
||||||
# 3. 获取已上传分片
|
# 4. 获取已上传分片
|
||||||
uploaded_info = self._list_uploaded_parts(drive_id=target_dir.drive_id, file_id=file_id, upload_id=upload_id)
|
uploaded_info = self._list_uploaded_parts(drive_id=target_dir.drive_id, file_id=file_id, upload_id=upload_id)
|
||||||
for part in uploaded_info.get('uploaded_parts', []):
|
for part in uploaded_info.get('uploaded_parts', []):
|
||||||
uploaded_parts.add(part['part_number'])
|
uploaded_parts[part['part_number']] = part.get('etag', '')
|
||||||
|
|
||||||
# 4. 初始化进度条
|
# 5. 初始化进度条
|
||||||
logger.info(f"【阿里云盘】开始上传: {local_path} -> {target_path},分片数:{len(part_info_list)}")
|
logger.info(f"【阿里云盘】开始上传: {local_path} -> {target_path},"
|
||||||
progress_bar = tqdm(
|
f"分片大小:{StringUtils.str_filesize(part_size)},线程数:{workers}")
|
||||||
total=file_size,
|
progress_bar = self._log_progress(f"【阿里云盘】{target_name} 上传进度", file_size)
|
||||||
unit='B',
|
|
||||||
unit_scale=True,
|
|
||||||
desc="上传进度",
|
|
||||||
ascii=True
|
|
||||||
)
|
|
||||||
|
|
||||||
# 5. 分片上传循环
|
# 7. 创建线程池
|
||||||
with open(local_path, 'rb') as f:
|
with ThreadPoolExecutor(max_workers=workers) as pool:
|
||||||
for part_info in part_info_list:
|
futures = []
|
||||||
part_num = part_info['part_number']
|
|
||||||
|
|
||||||
# 计算分片参数
|
# 提交上传任务
|
||||||
start = (part_num - 1) * chunk_size
|
with open(local_path, 'rb') as f:
|
||||||
end = min(start + chunk_size, file_size)
|
for part_info in part_info_list:
|
||||||
current_chunk_size = end - start
|
part_num = part_info['part_number']
|
||||||
|
|
||||||
# 更新进度条(已存在的分片)
|
# 跳过已上传的分片
|
||||||
if part_num in uploaded_parts:
|
if part_num in uploaded_parts:
|
||||||
progress_bar.update(current_chunk_size)
|
start = (part_num - 1) * part_size
|
||||||
continue
|
end = min(start + part_size, file_size)
|
||||||
|
progress_bar.update(end - start)
|
||||||
|
continue
|
||||||
|
|
||||||
# 准备分片数据
|
# 准备分片数据
|
||||||
f.seek(start)
|
start = (part_num - 1) * part_size
|
||||||
data = f.read(current_chunk_size)
|
end = min(start + part_size, file_size)
|
||||||
|
f.seek(start)
|
||||||
|
data = f.read(end - start)
|
||||||
|
|
||||||
# 上传分片(带重试逻辑)
|
# 提交上传任务
|
||||||
success = False
|
future = pool.submit(
|
||||||
for attempt in range(3): # 最大重试次数
|
self._upload_part,
|
||||||
try:
|
part_info['upload_url'],
|
||||||
# 获取当前上传地址(可能刷新)
|
data,
|
||||||
if attempt > 0:
|
part_num
|
||||||
new_urls = self._refresh_upload_urls(drive_id=target_dir.drive_id, file_id=file_id,
|
)
|
||||||
upload_id=upload_id, part_numbers=[part_num])
|
futures.append((part_num, future))
|
||||||
upload_url = new_urls[0]['upload_url']
|
|
||||||
else:
|
|
||||||
upload_url = part_info['upload_url']
|
|
||||||
|
|
||||||
# 执行上传
|
# 等待所有任务完成
|
||||||
logger.info(
|
for part_num, future in futures:
|
||||||
f"【阿里云盘】开始 第{attempt + 1}次 上传 {target_name} 分片 {part_num} ...")
|
try:
|
||||||
response = self._upload_part(upload_url=upload_url, data=data)
|
num, etag, uploaded = future.result()
|
||||||
if response is None:
|
uploaded_parts[num] = etag
|
||||||
continue
|
progress_bar.update(uploaded)
|
||||||
if response.status_code == 200:
|
except Exception as e:
|
||||||
success = True
|
logger.error(f"【阿里云盘】分片上传失败: {str(e)}")
|
||||||
break
|
progress_bar.close()
|
||||||
else:
|
return None
|
||||||
logger.warn(
|
|
||||||
f"【阿里云盘】{target_name} 分片 {part_num} 第 {attempt + 1} 次上传失败:{response.text}!")
|
|
||||||
except Exception as e:
|
|
||||||
logger.warn(f"【阿里云盘】{target_name} 分片 {part_num} 上传异常: {str(e)}!")
|
|
||||||
|
|
||||||
# 处理上传结果
|
# 8. 关闭进度条
|
||||||
if success:
|
progress_bar.close()
|
||||||
uploaded_parts.add(part_num)
|
|
||||||
progress_bar.update(current_chunk_size)
|
|
||||||
else:
|
|
||||||
raise Exception(f"【阿里云盘】{target_name} 分片 {part_num} 上传失败!")
|
|
||||||
|
|
||||||
# 6. 关闭进度条
|
# 9. 完成上传
|
||||||
if progress_bar:
|
|
||||||
progress_bar.close()
|
|
||||||
|
|
||||||
# 7. 完成上传
|
|
||||||
result = self._complete_upload(drive_id=target_dir.drive_id, file_id=file_id, upload_id=upload_id)
|
result = self._complete_upload(drive_id=target_dir.drive_id, file_id=file_id, upload_id=upload_id)
|
||||||
if not result:
|
if not result:
|
||||||
raise Exception("【阿里云盘】完成上传失败!")
|
raise Exception("【阿里云盘】完成上传失败!")
|
||||||
if result.get("code"):
|
if result.get("code"):
|
||||||
logger.warn(f"【阿里云盘】{target_name} 上传失败:{result.get('message')}!")
|
logger.warn(f"【阿里云盘】{target_name} 上传失败:{result.get('message')}!")
|
||||||
|
return None
|
||||||
|
|
||||||
return self.__get_fileitem(result, parent=target_dir.path)
|
return self.__get_fileitem(result, parent=target_dir.path)
|
||||||
|
|
||||||
def download(self, fileitem: schemas.FileItem, path: Path = None) -> Optional[Path]:
|
def download(self, fileitem: schemas.FileItem, path: Path = None) -> Optional[Path]:
|
||||||
@@ -828,7 +876,7 @@ class AliPan(StorageBase, metaclass=Singleton):
|
|||||||
if resp.get("code"):
|
if resp.get("code"):
|
||||||
logger.debug(f"【阿里云盘】获取文件信息失败: {resp.get('message')}")
|
logger.debug(f"【阿里云盘】获取文件信息失败: {resp.get('message')}")
|
||||||
return None
|
return None
|
||||||
return self.__get_fileitem(resp, parent=f"{str(path.parent)}/")
|
return self.__get_fileitem(resp, parent=str(path.parent))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug(f"【阿里云盘】获取文件信息失败: {str(e)}")
|
logger.debug(f"【阿里云盘】获取文件信息失败: {str(e)}")
|
||||||
return None
|
return None
|
||||||
@@ -854,7 +902,7 @@ class AliPan(StorageBase, metaclass=Singleton):
|
|||||||
if folder:
|
if folder:
|
||||||
return folder
|
return folder
|
||||||
# 逐级查找和创建目录
|
# 逐级查找和创建目录
|
||||||
fileitem = schemas.FileItem(storage=self.schema.value, path="/")
|
fileitem = schemas.FileItem(storage=self.schema.value, path="/", drive_id=self._default_drive_id)
|
||||||
for part in path.parts[1:]:
|
for part in path.parts[1:]:
|
||||||
dir_file = __find_dir(fileitem, part)
|
dir_file = __find_dir(fileitem, part)
|
||||||
if dir_file:
|
if dir_file:
|
||||||
@@ -957,3 +1005,5 @@ class AliPan(StorageBase, metaclass=Singleton):
|
|||||||
)
|
)
|
||||||
except NoCheckInException:
|
except NoCheckInException:
|
||||||
return None
|
return None
|
||||||
|
except SessionInvalidException:
|
||||||
|
return None
|
||||||
|
|||||||
@@ -1,15 +1,15 @@
|
|||||||
import base64
|
import base64
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import io
|
||||||
import secrets
|
import secrets
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import List, Dict, Optional, Tuple, Union
|
from typing import List, Dict, Optional, Tuple, Union
|
||||||
|
|
||||||
import oss2
|
import oss2
|
||||||
import requests
|
import requests
|
||||||
from oss2 import SizedFileAdapter, determine_part_size
|
|
||||||
from oss2.models import PartInfo
|
from oss2.models import PartInfo
|
||||||
from tqdm import tqdm
|
from tqdm import tqdm
|
||||||
|
|
||||||
@@ -54,6 +54,13 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
|||||||
# CID和路径缓存
|
# CID和路径缓存
|
||||||
_id_cache: Dict[str, str] = {}
|
_id_cache: Dict[str, str] = {}
|
||||||
|
|
||||||
|
# 最大线程数
|
||||||
|
MAX_WORKERS = 10
|
||||||
|
# 最大分片大小(1GB)
|
||||||
|
MAX_PART_SIZE = 1024 * 1024 * 1024
|
||||||
|
# 最小分片大小(100MB)
|
||||||
|
MIN_PART_SIZE = 100 * 1024 * 1024
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.session = requests.Session()
|
self.session = requests.Session()
|
||||||
@@ -375,7 +382,7 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
|||||||
"POST",
|
"POST",
|
||||||
"/open/folder/add",
|
"/open/folder/add",
|
||||||
data={
|
data={
|
||||||
"pid": int(parent_item.fileid),
|
"pid": int(parent_item.fileid or "0"),
|
||||||
"file_name": name
|
"file_name": name
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
@@ -399,17 +406,66 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
|||||||
modify_time=int(time.time())
|
modify_time=int(time.time())
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _calc_parts(self, file_size: int) -> Tuple[int, int]:
|
||||||
|
"""
|
||||||
|
计算最优分片大小和线程数,在最大分片大小和最小分片大小之间取最优值
|
||||||
|
:param file_size: 文件大小
|
||||||
|
:return: 分片大小,线程数
|
||||||
|
"""
|
||||||
|
if file_size <= self.MIN_PART_SIZE:
|
||||||
|
return file_size, 1
|
||||||
|
if file_size >= self.MAX_PART_SIZE:
|
||||||
|
part_size = self.MAX_PART_SIZE
|
||||||
|
else:
|
||||||
|
part_size = max(self.MIN_PART_SIZE, file_size // self.MAX_WORKERS)
|
||||||
|
return part_size, (file_size + part_size - 1) // part_size
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _upload_part(bucket: oss2.Bucket, object_name: str, upload_id: str,
|
||||||
|
part_number: int, part_data: bytes) -> Tuple[PartInfo, int]:
|
||||||
|
"""
|
||||||
|
上传单个分片
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
result = bucket.upload_part(object_name, upload_id, part_number, part_data)
|
||||||
|
part_info = PartInfo(part_number, result.etag)
|
||||||
|
logger.info(f"【115】分片 {part_number} 上传完成")
|
||||||
|
return part_info, len(part_data)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"【115】分片 {part_number} 上传失败: {str(e)}")
|
||||||
|
raise
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _log_progress(desc: str, total: int) -> tqdm:
|
||||||
|
"""
|
||||||
|
创建一个可以输出到日志的进度条
|
||||||
|
"""
|
||||||
|
|
||||||
|
class TqdmToLogger(io.StringIO):
|
||||||
|
def write(s, buf): # noqa
|
||||||
|
buf = buf.strip('\r\n\t ')
|
||||||
|
if buf:
|
||||||
|
logger.info(buf)
|
||||||
|
|
||||||
|
return tqdm(
|
||||||
|
total=total,
|
||||||
|
unit='B',
|
||||||
|
unit_scale=True,
|
||||||
|
desc=desc,
|
||||||
|
file=TqdmToLogger(),
|
||||||
|
mininterval=1.0,
|
||||||
|
maxinterval=5.0,
|
||||||
|
miniters=1
|
||||||
|
)
|
||||||
|
|
||||||
def upload(self, target_dir: schemas.FileItem, local_path: Path,
|
def upload(self, target_dir: schemas.FileItem, local_path: Path,
|
||||||
new_name: Optional[str] = None) -> Optional[schemas.FileItem]:
|
new_name: Optional[str] = None) -> Optional[schemas.FileItem]:
|
||||||
"""
|
"""
|
||||||
实现带秒传、断点续传和二次认证的文件上传
|
实现带秒传、断点续传和多线程并发上传
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def encode_callback(cb: dict):
|
def encode_callback(cb: str):
|
||||||
"""
|
return oss2.utils.b64encode_as_string(cb)
|
||||||
回调参数Base64编码函数
|
|
||||||
"""
|
|
||||||
return oss2.utils.b64encode_as_string(json.dumps(cb).strip())
|
|
||||||
|
|
||||||
target_name = new_name or local_path.name
|
target_name = new_name or local_path.name
|
||||||
target_path = Path(target_dir.path) / target_name
|
target_path = Path(target_dir.path) / target_name
|
||||||
@@ -440,6 +496,7 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
|||||||
if not init_resp.get("state"):
|
if not init_resp.get("state"):
|
||||||
logger.warn(f"【115】初始化上传失败: {init_resp.get('error')}")
|
logger.warn(f"【115】初始化上传失败: {init_resp.get('error')}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# 结果
|
# 结果
|
||||||
init_result = init_resp.get("data")
|
init_result = init_resp.get("data")
|
||||||
logger.debug(f"【115】上传 Step 1 初始化结果: {init_result}")
|
logger.debug(f"【115】上传 Step 1 初始化结果: {init_result}")
|
||||||
@@ -457,15 +514,10 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
|||||||
sign_checks = sign_check.split("-")
|
sign_checks = sign_check.split("-")
|
||||||
start = int(sign_checks[0])
|
start = int(sign_checks[0])
|
||||||
end = int(sign_checks[1])
|
end = int(sign_checks[1])
|
||||||
# 计算指定区间的SHA1
|
|
||||||
# sign_check (用下划线隔开,截取上传文内容的sha1)(单位是byte): "2392148-2392298"
|
|
||||||
with open(local_path, "rb") as f:
|
with open(local_path, "rb") as f:
|
||||||
# 取2392148-2392298之间的内容(包含2392148、2392298)的sha1
|
|
||||||
f.seek(start)
|
f.seek(start)
|
||||||
chunk = f.read(end - start + 1)
|
chunk = f.read(end - start + 1)
|
||||||
sign_val = hashlib.sha1(chunk).hexdigest().upper()
|
sign_val = hashlib.sha1(chunk).hexdigest().upper()
|
||||||
# 重新初始化请求
|
|
||||||
# sign_key,sign_val(根据sign_check计算的值大写的sha1值)
|
|
||||||
init_data.update({
|
init_data.update({
|
||||||
"pick_code": pick_code,
|
"pick_code": pick_code,
|
||||||
"sign_key": sign_key,
|
"sign_key": sign_key,
|
||||||
@@ -478,7 +530,6 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
|||||||
)
|
)
|
||||||
if not init_resp:
|
if not init_resp:
|
||||||
return None
|
return None
|
||||||
# 二次认证结果
|
|
||||||
init_result = init_resp.get("data")
|
init_result = init_resp.get("data")
|
||||||
logger.debug(f"【115】上传 Step 2 二次认证结果: {init_result}")
|
logger.debug(f"【115】上传 Step 2 二次认证结果: {init_result}")
|
||||||
if not pick_code:
|
if not pick_code:
|
||||||
@@ -505,7 +556,7 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
|||||||
logger.warn("【115】获取上传凭证失败")
|
logger.warn("【115】获取上传凭证失败")
|
||||||
return None
|
return None
|
||||||
logger.debug(f"【115】上传 Step 4 获取上传凭证结果: {token_resp}")
|
logger.debug(f"【115】上传 Step 4 获取上传凭证结果: {token_resp}")
|
||||||
# 上传凭证
|
|
||||||
endpoint = token_resp.get("endpoint")
|
endpoint = token_resp.get("endpoint")
|
||||||
AccessKeyId = token_resp.get("AccessKeyId")
|
AccessKeyId = token_resp.get("AccessKeyId")
|
||||||
AccessKeySecret = token_resp.get("AccessKeySecret")
|
AccessKeySecret = token_resp.get("AccessKeySecret")
|
||||||
@@ -528,66 +579,75 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
|||||||
if resume_resp.get("callback"):
|
if resume_resp.get("callback"):
|
||||||
callback = resume_resp["callback"]
|
callback = resume_resp["callback"]
|
||||||
|
|
||||||
# Step 6: 对象存储上传
|
# Step 6: 多线程分片上传
|
||||||
auth = oss2.StsAuth(
|
auth = oss2.StsAuth(
|
||||||
access_key_id=AccessKeyId,
|
access_key_id=AccessKeyId,
|
||||||
access_key_secret=AccessKeySecret,
|
access_key_secret=AccessKeySecret,
|
||||||
security_token=SecurityToken
|
security_token=SecurityToken
|
||||||
)
|
)
|
||||||
bucket = oss2.Bucket(auth, endpoint, bucket_name) # noqa
|
bucket = oss2.Bucket(auth, endpoint, bucket_name) # noqa
|
||||||
# 处理oss请求回调
|
|
||||||
callback_dict = json.loads(callback.get("callback"))
|
# 计算分片大小和线程数
|
||||||
callback_var_dict = json.loads(callback.get("callback_var"))
|
part_size, workers = self._calc_parts(file_size)
|
||||||
# 补充参数
|
logger.info(f"【115】开始上传: {local_path} -> {target_path},"
|
||||||
logger.debug(f"【115】上传 Step 6 回调参数:{callback_dict} {callback_var_dict}")
|
f"分片大小:{StringUtils.str_filesize(part_size)},线程数:{workers}")
|
||||||
# 填写不能包含Bucket名称在内的Object完整路径,例如exampledir/exampleobject.txt。
|
|
||||||
# determine_part_size方法用于确定分片大小,设置分片大小为 100M
|
|
||||||
part_size = determine_part_size(file_size, preferred_size=100 * 1024 * 1024)
|
|
||||||
|
|
||||||
# 初始化进度条
|
# 初始化进度条
|
||||||
logger.info(f"【115】开始上传: {local_path} -> {target_path},分片大小:{StringUtils.str_filesize(part_size)}")
|
progress_bar = self._log_progress(f"【115】{target_name} 上传进度", file_size)
|
||||||
progress_bar = tqdm(
|
|
||||||
total=file_size,
|
|
||||||
unit='B',
|
|
||||||
unit_scale=True,
|
|
||||||
desc="上传进度",
|
|
||||||
ascii=True
|
|
||||||
)
|
|
||||||
|
|
||||||
# 初始化分片
|
# 初始化分片上传
|
||||||
upload_id = bucket.init_multipart_upload(object_name,
|
upload_id = bucket.init_multipart_upload(object_name,
|
||||||
params={
|
params={
|
||||||
"encoding-type": "url",
|
"encoding-type": "url",
|
||||||
"sequential": ""
|
"sequential": ""
|
||||||
}).upload_id
|
}).upload_id
|
||||||
parts = []
|
|
||||||
# 逐个上传分片
|
|
||||||
with open(local_path, 'rb') as fileobj:
|
|
||||||
part_number = 1
|
|
||||||
offset = 0
|
|
||||||
while offset < file_size:
|
|
||||||
num_to_upload = min(part_size, file_size - offset)
|
|
||||||
# 调用SizedFileAdapter(fileobj, size)方法会生成一个新的文件对象,重新计算起始追加位置。
|
|
||||||
logger.info(f"【115】开始上传 {target_name} 分片 {part_number}: {offset} -> {offset + num_to_upload}")
|
|
||||||
result = bucket.upload_part(object_name, upload_id, part_number,
|
|
||||||
data=SizedFileAdapter(fileobj, num_to_upload))
|
|
||||||
parts.append(PartInfo(part_number, result.etag))
|
|
||||||
logger.info(f"【115】{target_name} 分片 {part_number} 上传完成")
|
|
||||||
offset += num_to_upload
|
|
||||||
part_number += 1
|
|
||||||
# 更新进度
|
|
||||||
progress_bar.update(num_to_upload)
|
|
||||||
|
|
||||||
# 关闭进度条
|
# 创建线程池
|
||||||
if progress_bar:
|
with ThreadPoolExecutor(max_workers=workers) as pool:
|
||||||
progress_bar.close()
|
futures = []
|
||||||
|
parts = []
|
||||||
|
|
||||||
# 请求头
|
# 提交上传任务
|
||||||
|
with open(local_path, 'rb') as fileobj:
|
||||||
|
part_number = 1
|
||||||
|
offset = 0
|
||||||
|
while offset < file_size:
|
||||||
|
size = min(part_size, file_size - offset)
|
||||||
|
fileobj.seek(offset)
|
||||||
|
part_data = fileobj.read(size)
|
||||||
|
future = pool.submit(
|
||||||
|
self._upload_part,
|
||||||
|
bucket,
|
||||||
|
object_name,
|
||||||
|
upload_id,
|
||||||
|
part_number,
|
||||||
|
part_data
|
||||||
|
)
|
||||||
|
futures.append(future)
|
||||||
|
offset += size
|
||||||
|
part_number += 1
|
||||||
|
|
||||||
|
# 等待所有任务完成
|
||||||
|
for future in as_completed(futures):
|
||||||
|
try:
|
||||||
|
part_info, uploaded = future.result()
|
||||||
|
parts.append(part_info)
|
||||||
|
progress_bar.update(uploaded)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"【115】分片上传失败: {str(e)}")
|
||||||
|
progress_bar.close()
|
||||||
|
return None
|
||||||
|
|
||||||
|
# 按分片号排序
|
||||||
|
parts.sort(key=lambda x: x.part_number)
|
||||||
|
|
||||||
|
# 完成上传
|
||||||
headers = {
|
headers = {
|
||||||
'X-oss-callback': encode_callback(callback_dict),
|
'X-oss-callback': encode_callback(callback["callback"]),
|
||||||
'x-oss-callback-var': encode_callback(callback_var_dict),
|
'x-oss-callback-var': encode_callback(callback["callback_var"]),
|
||||||
'x-oss-forbid-overwrite': 'false'
|
'x-oss-forbid-overwrite': 'false'
|
||||||
}
|
}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = bucket.complete_multipart_upload(object_name, upload_id, parts,
|
result = bucket.complete_multipart_upload(object_name, upload_id, parts,
|
||||||
headers=headers)
|
headers=headers)
|
||||||
@@ -603,6 +663,9 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
|||||||
else:
|
else:
|
||||||
logger.error(f"【115】{target_name} 上传失败: {e.status}, 错误码: {e.code}, 详情: {e.message}")
|
logger.error(f"【115】{target_name} 上传失败: {e.status}, 错误码: {e.code}, 详情: {e.message}")
|
||||||
return None
|
return None
|
||||||
|
finally:
|
||||||
|
progress_bar.close()
|
||||||
|
|
||||||
# 返回结果
|
# 返回结果
|
||||||
return self.get_item(target_path)
|
return self.get_item(target_path)
|
||||||
|
|
||||||
|
|||||||
@@ -44,6 +44,7 @@ class TrimeMedia:
|
|||||||
self._playhost = play_api.host
|
self._playhost = play_api.host
|
||||||
elif play_host:
|
elif play_host:
|
||||||
logger.warning(f"请检查外网播放地址 {play_host}")
|
logger.warning(f"请检查外网播放地址 {play_host}")
|
||||||
|
self._playhost = UrlUtils.standardize_base_url(play_host).rstrip("/")
|
||||||
|
|
||||||
self.reconnect()
|
self.reconnect()
|
||||||
|
|
||||||
|
|||||||
+11
-4
@@ -52,21 +52,28 @@ class ObjectUtils:
|
|||||||
# 跳过空行
|
# 跳过空行
|
||||||
if not line:
|
if not line:
|
||||||
continue
|
continue
|
||||||
# 处理多行注释
|
# 处理"""单行注释
|
||||||
|
if line.startswith(('"""', "'''")) and line.endswith(('"""', "'''")):
|
||||||
|
continue
|
||||||
|
# 处理"""多行注释
|
||||||
if line.startswith(('"""', "'''")):
|
if line.startswith(('"""', "'''")):
|
||||||
in_comment = not in_comment
|
in_comment = not in_comment
|
||||||
continue
|
continue
|
||||||
# 在注释中则跳过
|
# 在注释中则跳过
|
||||||
if in_comment:
|
if in_comment:
|
||||||
continue
|
continue
|
||||||
# 跳过注释、pass语句、装饰器、函数定义行
|
# 跳过#注释、pass语句、装饰器、函数定义行
|
||||||
if line.startswith('#') or line == "pass" or line.startswith('@') or line.startswith('def '):
|
if (line.startswith('#')
|
||||||
|
or line == "pass"
|
||||||
|
or line.startswith('@')
|
||||||
|
or line.startswith('def ')):
|
||||||
continue
|
continue
|
||||||
# 发现有效代码行
|
# 发现有效代码行
|
||||||
return True
|
return True
|
||||||
# 没有有效代码行
|
# 没有有效代码行
|
||||||
return False
|
return False
|
||||||
except Exception:
|
except Exception as err:
|
||||||
|
print(err)
|
||||||
# 源代码分析失败时,进行字节码分析
|
# 源代码分析失败时,进行字节码分析
|
||||||
code_obj = func.__code__
|
code_obj = func.__code__
|
||||||
instructions = list(dis.get_instructions(code_obj))
|
instructions = list(dis.get_instructions(code_obj))
|
||||||
|
|||||||
+2
-2
@@ -1,2 +1,2 @@
|
|||||||
APP_VERSION = 'v2.4.4-1'
|
APP_VERSION = 'v2.4.5'
|
||||||
FRONTEND_VERSION = 'v2.4.4-1'
|
FRONTEND_VERSION = 'v2.4.5'
|
||||||
|
|||||||
Reference in New Issue
Block a user