mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-05-10 17:12:39 +08:00
Compare commits
133 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5340e3a0a7 | ||
|
|
70dd8f0f1d | ||
|
|
8fa76504c3 | ||
|
|
0899cb4e1d | ||
|
|
ee7a2a70a6 | ||
|
|
d57d1ac15e | ||
|
|
68c29d89c9 | ||
|
|
721648ffdf | ||
|
|
8437f39bf6 | ||
|
|
48b15c60e7 | ||
|
|
e350122125 | ||
|
|
0cce97f373 | ||
|
|
d8cacc0811 | ||
|
|
7abaf70bb8 | ||
|
|
232fe4d15e | ||
|
|
d6d12c0335 | ||
|
|
8e4f12804b | ||
|
|
c21ba5c521 | ||
|
|
dfa3d47261 | ||
|
|
924f59afff | ||
|
|
673b282d6c | ||
|
|
1c761f89e5 | ||
|
|
f61cd969b9 | ||
|
|
e39a130306 | ||
|
|
13b6ea985e | ||
|
|
2f1e55fa1e | ||
|
|
776f629771 | ||
|
|
d9e9edb2c4 | ||
|
|
753c074e59 | ||
|
|
d92c82775a | ||
|
|
215cc09c1f | ||
|
|
7f302c13c7 | ||
|
|
de6a094d10 | ||
|
|
a94e1a8314 | ||
|
|
f5efdd665b | ||
|
|
43e25e8717 | ||
|
|
a8026fefc1 | ||
|
|
fdb36957c9 | ||
|
|
ea433ff807 | ||
|
|
8902fb50d6 | ||
|
|
b6aa013eb3 | ||
|
|
034b43bf70 | ||
|
|
59e9032286 | ||
|
|
52a98efd0a | ||
|
|
90cc91aa7f | ||
|
|
1973a26e83 | ||
|
|
6519ad25ca | ||
|
|
cacfde8166 | ||
|
|
df85873726 | ||
|
|
dfea294cc9 | ||
|
|
d35b855404 | ||
|
|
7a1cbf70e3 | ||
|
|
f260990b86 | ||
|
|
6affbe9b55 | ||
|
|
dbe3a10697 | ||
|
|
3c25306a5d | ||
|
|
17f4d49731 | ||
|
|
e213b5cc64 | ||
|
|
65e5dad44b | ||
|
|
62ad38ea5d | ||
|
|
f98f4c1f77 | ||
|
|
e9f02b58b7 | ||
|
|
05495e481d | ||
|
|
5bb2167b78 | ||
|
|
b4e0ed66cf | ||
|
|
70a0563435 | ||
|
|
955912b832 | ||
|
|
b65ee75b3d | ||
|
|
f642493a38 | ||
|
|
7f1bfb1e07 | ||
|
|
8931e2e016 | ||
|
|
0465fa77c2 | ||
|
|
575d503cb9 | ||
|
|
a4fdbdb9ad | ||
|
|
b9cb781a4e | ||
|
|
a3adf867b7 | ||
|
|
d52cbd2f74 | ||
|
|
8d0003db94 | ||
|
|
b775e89e77 | ||
|
|
0e14b097ba | ||
|
|
51848b8d8d | ||
|
|
72658c3e60 | ||
|
|
036cb6f3b0 | ||
|
|
1a86d96bfa | ||
|
|
f67db38a25 | ||
|
|
028d18826a | ||
|
|
29a605f265 | ||
|
|
4b6959470d | ||
|
|
600767d2bf | ||
|
|
3efbd47ffd | ||
|
|
d17e85217b | ||
|
|
e608089805 | ||
|
|
b852acec28 | ||
|
|
2a3ea8315d | ||
|
|
9271ee833c | ||
|
|
570d4ad1a3 | ||
|
|
dccdf3231a | ||
|
|
b8ee777fd2 | ||
|
|
a2fd3a8d90 | ||
|
|
bbffb1420b | ||
|
|
8ea0a32879 | ||
|
|
8c27b8c33e | ||
|
|
5c61b22c2f | ||
|
|
9da9d765a0 | ||
|
|
f64363728e | ||
|
|
378777dc7c | ||
|
|
6156b9a481 | ||
|
|
8c516c5691 | ||
|
|
bf9a149898 | ||
|
|
277cde8db2 | ||
|
|
e06bdaf53e | ||
|
|
da367bd138 | ||
|
|
d336bcbf1f | ||
|
|
a8aedba6ff | ||
|
|
9ede86c6a3 | ||
|
|
1468f2b082 | ||
|
|
e04ae70f89 | ||
|
|
7f7d2c9ba8 | ||
|
|
d73deef8dc | ||
|
|
f93a1540af | ||
|
|
c8bd9cb716 | ||
|
|
2ed13c7e5b | ||
|
|
647c0929c5 | ||
|
|
a61533a131 | ||
|
|
bc5e682308 | ||
|
|
25a481df12 | ||
|
|
764c10fae4 | ||
|
|
d8249d4e38 | ||
|
|
0e3e42b398 | ||
|
|
7d3b64dcf9 | ||
|
|
2c8d525796 | ||
|
|
4869f071ab | ||
|
|
3029eeaf6f |
30
app/actions/note.py
Normal file
30
app/actions/note.py
Normal file
@@ -0,0 +1,30 @@
|
||||
from app.actions import BaseAction
|
||||
from app.schemas import ActionContext
|
||||
|
||||
|
||||
class NoteAction(BaseAction):
|
||||
"""
|
||||
备注
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
@property
|
||||
def name(cls) -> str: # noqa
|
||||
return "备注"
|
||||
|
||||
@classmethod
|
||||
@property
|
||||
def description(cls) -> str: # noqa
|
||||
return "给工作流添加备注"
|
||||
|
||||
@classmethod
|
||||
@property
|
||||
def data(cls) -> dict: # noqa
|
||||
return {}
|
||||
|
||||
@property
|
||||
def success(self) -> bool:
|
||||
return True
|
||||
|
||||
def execute(self, workflow_id: int, params: dict, context: ActionContext) -> ActionContext:
|
||||
return context
|
||||
@@ -1,6 +1,5 @@
|
||||
from typing import List, Any, Dict, Optional
|
||||
|
||||
from app.helper.sites import SitesHelper
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
from starlette.background import BackgroundTasks
|
||||
@@ -22,6 +21,7 @@ from app.db.models.siteuserdata import SiteUserData
|
||||
from app.db.site_oper import SiteOper
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.db.user_oper import get_current_active_superuser
|
||||
from app.helper.sites import SitesHelper
|
||||
from app.scheduler import Scheduler
|
||||
from app.schemas.types import SystemConfigKey, EventType
|
||||
from app.utils.string import StringUtils
|
||||
@@ -422,6 +422,14 @@ def site_mapping(_: User = Depends(get_current_active_superuser)):
|
||||
return schemas.Response(success=False, message=f"获取映射失败:{str(e)}")
|
||||
|
||||
|
||||
@router.get("/supporting", summary="获取支持的站点列表", response_model=dict)
|
||||
def support_sites(_: User = Depends(get_current_active_superuser)):
|
||||
"""
|
||||
获取支持的站点列表
|
||||
"""
|
||||
return SitesHelper().get_indexsites()
|
||||
|
||||
|
||||
@router.get("/{site_id}", summary="站点详情", response_model=schemas.Site)
|
||||
def read_site(
|
||||
site_id: int,
|
||||
|
||||
@@ -560,6 +560,15 @@ def popular_subscribes(
|
||||
return SubscribeHelper().get_shares(name=name, page=page, count=count)
|
||||
|
||||
|
||||
@router.get("/share/statistics", summary="查询订阅分享统计", response_model=List[schemas.SubscribeShareStatistics])
|
||||
def subscribe_share_statistics(_: schemas.TokenPayload = Depends(verify_token)) -> Any:
|
||||
"""
|
||||
查询订阅分享统计
|
||||
返回每个分享人分享的媒体数量以及总的复用人次
|
||||
"""
|
||||
return SubscribeHelper().get_share_statistics()
|
||||
|
||||
|
||||
@router.get("/{subscribe_id}", summary="订阅详情", response_model=schemas.Subscribe)
|
||||
def read_subscribe(
|
||||
subscribe_id: int,
|
||||
|
||||
@@ -11,6 +11,7 @@ from typing import Optional, Union, Annotated
|
||||
import aiofiles
|
||||
import pillow_avif # noqa 用于自动注册AVIF支持
|
||||
from PIL import Image
|
||||
from app.helper.sites import SitesHelper
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Header, Request, Response
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
@@ -18,10 +19,10 @@ from app import schemas
|
||||
from app.chain.search import SearchChain
|
||||
from app.chain.system import SystemChain
|
||||
from app.core.config import global_vars, settings
|
||||
from app.core.event import eventmanager
|
||||
from app.core.metainfo import MetaInfo
|
||||
from app.core.module import ModuleManager
|
||||
from app.core.security import verify_apitoken, verify_resource_token, verify_token
|
||||
from app.core.event import eventmanager
|
||||
from app.db.models import User
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.db.user_oper import get_current_active_superuser
|
||||
@@ -29,7 +30,6 @@ from app.helper.mediaserver import MediaServerHelper
|
||||
from app.helper.message import MessageHelper
|
||||
from app.helper.progress import ProgressHelper
|
||||
from app.helper.rule import RuleHelper
|
||||
from app.helper.sites import SitesHelper
|
||||
from app.helper.subscribe import SubscribeHelper
|
||||
from app.helper.system import SystemHelper
|
||||
from app.log import logger
|
||||
@@ -95,7 +95,7 @@ def fetch_image(
|
||||
# 请求远程图片
|
||||
referer = "https://movie.douban.com/" if "doubanio.com" in url else None
|
||||
proxies = settings.PROXY if proxy else None
|
||||
response = RequestUtils(ua=settings.USER_AGENT, proxies=proxies, referer=referer,
|
||||
response = RequestUtils(ua=settings.NORMAL_USER_AGENT, proxies=proxies, referer=referer,
|
||||
accept_type="image/avif,image/webp,image/apng,*/*").get_res(url=url)
|
||||
if not response:
|
||||
raise HTTPException(status_code=502, detail="Failed to fetch the image from the remote server")
|
||||
@@ -187,9 +187,11 @@ def get_global_setting(token: str):
|
||||
"COOKIECLOUD_KEY", "COOKIECLOUD_PASSWORD", "GITHUB_TOKEN", "REPO_GITHUB_TOKEN"}
|
||||
)
|
||||
# 追加用户唯一ID和订阅分享管理权限
|
||||
share_admin = SubscribeHelper().is_admin_user()
|
||||
info.update({
|
||||
"USER_UNIQUE_ID": SubscribeHelper().get_user_uuid(),
|
||||
"SUBSCRIBE_SHARE_MANAGE": SubscribeHelper().is_admin_user(),
|
||||
"SUBSCRIBE_SHARE_MANAGE": share_admin,
|
||||
"WORKFLOW_SHARE_MANAGE": share_admin
|
||||
})
|
||||
return schemas.Response(success=True,
|
||||
data=info)
|
||||
@@ -290,9 +292,9 @@ def get_setting(key: str,
|
||||
|
||||
@router.post("/setting/{key}", summary="更新系统设置", response_model=schemas.Response)
|
||||
def set_setting(
|
||||
key: str,
|
||||
value: Annotated[Union[list, dict, bool, int, str] | None, Body()] = None,
|
||||
_: User = Depends(get_current_active_superuser),
|
||||
key: str,
|
||||
value: Annotated[Union[list, dict, bool, int, str] | None, Body()] = None,
|
||||
_: User = Depends(get_current_active_superuser),
|
||||
):
|
||||
"""
|
||||
更新系统设置(仅管理员)
|
||||
@@ -452,10 +454,10 @@ def ruletest(title: str,
|
||||
|
||||
@router.get("/nettest", summary="测试网络连通性")
|
||||
def nettest(
|
||||
url: str,
|
||||
proxy: bool,
|
||||
include: Optional[str] = None,
|
||||
_: schemas.TokenPayload = Depends(verify_token),
|
||||
url: str,
|
||||
proxy: bool,
|
||||
include: Optional[str] = None,
|
||||
_: schemas.TokenPayload = Depends(verify_token),
|
||||
):
|
||||
"""
|
||||
测试网络连通性
|
||||
@@ -463,43 +465,68 @@ def nettest(
|
||||
# 记录开始的毫秒数
|
||||
start_time = datetime.now()
|
||||
headers = None
|
||||
if "github" in url or "{GITHUB_PROXY}" in url:
|
||||
# 当前使用的加速代理
|
||||
proxy_name = ""
|
||||
if "github" in url:
|
||||
# 这是github的连通性测试
|
||||
headers = settings.GITHUB_HEADERS
|
||||
if "{GITHUB_PROXY}" in url:
|
||||
url = url.replace(
|
||||
"{GITHUB_PROXY}", UrlUtils.standardize_base_url(settings.GITHUB_PROXY or "")
|
||||
)
|
||||
headers = settings.GITHUB_HEADERS
|
||||
if settings.GITHUB_PROXY:
|
||||
proxy_name = "Github加速代理"
|
||||
if "{PIP_PROXY}" in url:
|
||||
url = url.replace(
|
||||
"{PIP_PROXY}",
|
||||
UrlUtils.standardize_base_url(
|
||||
settings.PIP_PROXY or "https://pypi.org/simple/"
|
||||
),
|
||||
)
|
||||
if settings.PIP_PROXY:
|
||||
proxy_name = "PIP加速代理"
|
||||
url = url.replace("{TMDBAPIKEY}", settings.TMDB_API_KEY)
|
||||
url = url.replace(
|
||||
"{PIP_PROXY}",
|
||||
UrlUtils.standardize_base_url(settings.PIP_PROXY or "https://pypi.org/simple/"),
|
||||
)
|
||||
result = RequestUtils(
|
||||
proxies=settings.PROXY if proxy else None,
|
||||
headers=headers,
|
||||
timeout=10,
|
||||
ua=settings.USER_AGENT,
|
||||
ua=settings.NORMAL_USER_AGENT,
|
||||
).get_res(url)
|
||||
# 计时结束的毫秒数
|
||||
end_time = datetime.now()
|
||||
time = round((end_time - start_time).total_seconds() * 1000)
|
||||
# 计算相关秒数
|
||||
if result is None:
|
||||
return schemas.Response(success=False, message="无法连接", data={"time": time})
|
||||
return schemas.Response(
|
||||
success=False, message=f"{proxy_name}无法连接", data={"time": time}
|
||||
)
|
||||
elif result.status_code == 200:
|
||||
if include and not re.search(r"%s" % include, result.text, re.IGNORECASE):
|
||||
# 通常是被加速代理跳转到其它页面了
|
||||
logger.error(f"{url} 的响应内容不匹配包含规则 {include}")
|
||||
if proxy_name:
|
||||
message = f"{proxy_name}已失效,请检查配置"
|
||||
else:
|
||||
message = f"无效响应,不匹配 {include}"
|
||||
return schemas.Response(
|
||||
success=False,
|
||||
message=f"无效响应,不匹配 {include}",
|
||||
message=message,
|
||||
data={"time": time},
|
||||
)
|
||||
return schemas.Response(success=True, data={"time": time})
|
||||
else:
|
||||
return schemas.Response(
|
||||
success=False, message=f"错误码:{result.status_code}", data={"time": time}
|
||||
)
|
||||
if proxy_name:
|
||||
# 加速代理失败
|
||||
message = f"{proxy_name}已失效,错误码:{result.status_code}"
|
||||
else:
|
||||
message = f"错误码:{result.status_code}"
|
||||
if "github" in url:
|
||||
# 非加速代理访问github
|
||||
if result.status_code == 401:
|
||||
message = "Github Token已失效,请检查配置"
|
||||
elif result.status_code in {403, 429}:
|
||||
message = "触发限流,请配置Github Token"
|
||||
return schemas.Response(success=False, message=message, data={"time": time})
|
||||
|
||||
|
||||
@router.get("/modulelist", summary="查询已加载的模块ID列表", response_model=schemas.Response)
|
||||
|
||||
@@ -8,11 +8,13 @@ from app import schemas
|
||||
from app.chain.media import MediaChain
|
||||
from app.chain.storage import StorageChain
|
||||
from app.chain.transfer import TransferChain
|
||||
from app.core.config import settings
|
||||
from app.core.metainfo import MetaInfoPath
|
||||
from app.core.security import verify_token, verify_apitoken
|
||||
from app.db import get_db
|
||||
from app.db.models.transferhistory import TransferHistory
|
||||
from app.db.user_oper import get_current_active_superuser
|
||||
from app.helper.directory import DirectoryHelper
|
||||
from app.schemas import MediaType, FileItem, ManualTransferItem
|
||||
|
||||
router = APIRouter()
|
||||
@@ -35,11 +37,19 @@ def query_name(path: str, filetype: str,
|
||||
if not new_path:
|
||||
return schemas.Response(success=False, message="未识别到新名称")
|
||||
if filetype == "dir":
|
||||
parents = Path(new_path).parents
|
||||
if len(parents) > 2:
|
||||
new_name = parents[1].name
|
||||
media_path = DirectoryHelper.get_media_root_path(
|
||||
rename_format=settings.RENAME_FORMAT(mediainfo.type),
|
||||
rename_path=Path(new_path),
|
||||
)
|
||||
if media_path:
|
||||
new_name = media_path.name
|
||||
else:
|
||||
new_name = parents[0].name
|
||||
# fallback
|
||||
parents = Path(new_path).parents
|
||||
if len(parents) > 2:
|
||||
new_name = parents[1].name
|
||||
else:
|
||||
new_name = parents[0].name
|
||||
else:
|
||||
new_name = Path(new_path).name
|
||||
return schemas.Response(success=True, data={
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import List, Any, Optional
|
||||
|
||||
@@ -5,15 +6,18 @@ from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import schemas
|
||||
from app.chain.workflow import WorkflowChain
|
||||
from app.core.config import global_vars
|
||||
from app.core.plugin import PluginManager
|
||||
from app.core.workflow import WorkFlowManager
|
||||
from app.db import get_db
|
||||
from app.db.models.workflow import Workflow
|
||||
from app.db.models import Workflow
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.db.user_oper import get_current_active_user
|
||||
from app.chain.workflow import WorkflowChain
|
||||
from app.db.workflow_oper import WorkflowOper
|
||||
from app.helper.workflow import WorkflowHelper
|
||||
from app.scheduler import Scheduler
|
||||
from app.schemas.types import EventType, EVENT_TYPE_NAMES
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -24,7 +28,8 @@ def list_workflows(db: Session = Depends(get_db),
|
||||
"""
|
||||
获取工作流列表
|
||||
"""
|
||||
return Workflow.list(db)
|
||||
from app.db.workflow_oper import WorkflowOper
|
||||
return WorkflowOper(db).list()
|
||||
|
||||
|
||||
@router.post("/", summary="创建工作流", response_model=schemas.Response)
|
||||
@@ -34,12 +39,14 @@ def create_workflow(workflow: schemas.Workflow,
|
||||
"""
|
||||
创建工作流
|
||||
"""
|
||||
if Workflow.get_by_name(db, workflow.name):
|
||||
if workflow.name and WorkflowOper(db).get_by_name(workflow.name):
|
||||
return schemas.Response(success=False, message="已存在相同名称的工作流")
|
||||
if not workflow.add_time:
|
||||
workflow.add_time = datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S")
|
||||
if not workflow.state:
|
||||
workflow.state = "P"
|
||||
if not workflow.trigger_type:
|
||||
workflow.trigger_type = "timer"
|
||||
Workflow(**workflow.dict()).create(db)
|
||||
return schemas.Response(success=True, message="创建工作流成功")
|
||||
|
||||
@@ -60,47 +67,111 @@ def list_actions(_: schemas.TokenPayload = Depends(get_current_active_user)) ->
|
||||
return WorkFlowManager().list_actions()
|
||||
|
||||
|
||||
@router.get("/{workflow_id}", summary="工作流详情", response_model=schemas.Workflow)
|
||||
def get_workflow(workflow_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
_: schemas.TokenPayload = Depends(get_current_active_user)) -> Any:
|
||||
@router.get("/event_types", summary="获取所有事件类型", response_model=List[dict])
|
||||
def get_event_types(_: schemas.TokenPayload = Depends(get_current_active_user)) -> Any:
|
||||
"""
|
||||
获取工作流详情
|
||||
获取所有事件类型
|
||||
"""
|
||||
return Workflow.get(db, workflow_id)
|
||||
return [{
|
||||
"title": EVENT_TYPE_NAMES.get(event_type, event_type.name),
|
||||
"value": event_type.value
|
||||
} for event_type in EventType]
|
||||
|
||||
|
||||
@router.put("/{workflow_id}", summary="更新工作流", response_model=schemas.Response)
|
||||
def update_workflow(workflow: schemas.Workflow,
|
||||
db: Session = Depends(get_db),
|
||||
_: schemas.TokenPayload = Depends(get_current_active_user)) -> Any:
|
||||
@router.post("/share", summary="分享工作流", response_model=schemas.Response)
|
||||
def workflow_share(
|
||||
workflow: schemas.WorkflowShare,
|
||||
_: schemas.TokenPayload = Depends(get_current_active_user)) -> Any:
|
||||
"""
|
||||
更新工作流
|
||||
分享工作流
|
||||
"""
|
||||
wf = Workflow.get(db, workflow.id)
|
||||
if not wf:
|
||||
return schemas.Response(success=False, message="工作流不存在")
|
||||
wf.update(db, workflow.dict())
|
||||
return schemas.Response(success=True, message="更新成功")
|
||||
if not workflow.id or not workflow.share_title or not workflow.share_user:
|
||||
return schemas.Response(success=False, message="请填写工作流ID、分享标题和分享人")
|
||||
|
||||
state, errmsg = WorkflowHelper().workflow_share(workflow_id=workflow.id,
|
||||
share_title=workflow.share_title or "",
|
||||
share_comment=workflow.share_comment or "",
|
||||
share_user=workflow.share_user or "")
|
||||
return schemas.Response(success=state, message=errmsg)
|
||||
|
||||
|
||||
@router.delete("/{workflow_id}", summary="删除工作流", response_model=schemas.Response)
|
||||
def delete_workflow(workflow_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
_: schemas.TokenPayload = Depends(get_current_active_user)) -> Any:
|
||||
@router.delete("/share/{share_id}", summary="删除分享", response_model=schemas.Response)
|
||||
def workflow_share_delete(
|
||||
share_id: int,
|
||||
_: schemas.TokenPayload = Depends(get_current_active_user)) -> Any:
|
||||
"""
|
||||
删除工作流
|
||||
删除分享
|
||||
"""
|
||||
workflow = Workflow.get(db, workflow_id)
|
||||
if not workflow:
|
||||
return schemas.Response(success=False, message="工作流不存在")
|
||||
# 删除定时任务
|
||||
Scheduler().remove_workflow_job(workflow)
|
||||
# 删除工作流
|
||||
Workflow.delete(db, workflow_id)
|
||||
# 删除缓存
|
||||
SystemConfigOper().delete(f"WorkflowCache-{workflow_id}")
|
||||
return schemas.Response(success=True, message="删除成功")
|
||||
state, errmsg = WorkflowHelper().share_delete(share_id=share_id)
|
||||
return schemas.Response(success=state, message=errmsg)
|
||||
|
||||
|
||||
@router.post("/fork", summary="复用工作流", response_model=schemas.Response)
|
||||
def workflow_fork(
|
||||
workflow: schemas.WorkflowShare,
|
||||
db: Session = Depends(get_db),
|
||||
_: schemas.User = Depends(get_current_active_user)) -> Any:
|
||||
"""
|
||||
复用工作流
|
||||
"""
|
||||
if not workflow.name:
|
||||
return schemas.Response(success=False, message="工作流名称不能为空")
|
||||
|
||||
# 解析JSON数据,添加错误处理
|
||||
try:
|
||||
actions = json.loads(workflow.actions or "[]")
|
||||
except json.JSONDecodeError:
|
||||
return schemas.Response(success=False, message="actions字段JSON格式错误")
|
||||
|
||||
try:
|
||||
flows = json.loads(workflow.flows or "[]")
|
||||
except json.JSONDecodeError:
|
||||
return schemas.Response(success=False, message="flows字段JSON格式错误")
|
||||
|
||||
try:
|
||||
context = json.loads(workflow.context or "{}")
|
||||
except json.JSONDecodeError:
|
||||
return schemas.Response(success=False, message="context字段JSON格式错误")
|
||||
|
||||
# 创建工作流
|
||||
workflow_dict = {
|
||||
"name": workflow.name,
|
||||
"description": workflow.description,
|
||||
"timer": workflow.timer,
|
||||
"trigger_type": workflow.trigger_type or "timer",
|
||||
"event_type": workflow.event_type,
|
||||
"event_conditions": json.loads(workflow.event_conditions or "{}") if workflow.event_conditions else {},
|
||||
"actions": actions,
|
||||
"flows": flows,
|
||||
"context": context,
|
||||
"state": "P" # 默认暂停状态
|
||||
}
|
||||
|
||||
# 检查名称是否重复
|
||||
if Workflow.get_by_name(db, workflow_dict["name"]):
|
||||
return schemas.Response(success=False, message="已存在相同名称的工作流")
|
||||
|
||||
# 创建新工作流
|
||||
workflow = Workflow(**workflow_dict)
|
||||
workflow.create(db)
|
||||
|
||||
# 更新复用次数
|
||||
if workflow.id:
|
||||
WorkflowHelper().workflow_fork(share_id=workflow.id)
|
||||
|
||||
return schemas.Response(success=True, message="复用成功")
|
||||
|
||||
|
||||
@router.get("/shares", summary="查询分享的工作流", response_model=List[schemas.WorkflowShare])
|
||||
def workflow_shares(
|
||||
name: Optional[str] = None,
|
||||
page: Optional[int] = 1,
|
||||
count: Optional[int] = 30,
|
||||
_: schemas.TokenPayload = Depends(get_current_active_user)) -> Any:
|
||||
"""
|
||||
查询分享的工作流
|
||||
"""
|
||||
return WorkflowHelper().get_shares(name=name, page=page, count=count)
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/run", summary="执行工作流", response_model=schemas.Response)
|
||||
@@ -123,11 +194,16 @@ def start_workflow(workflow_id: int,
|
||||
"""
|
||||
启用工作流
|
||||
"""
|
||||
workflow = Workflow.get(db, workflow_id)
|
||||
from app.db.workflow_oper import WorkflowOper
|
||||
workflow = WorkflowOper(db).get(workflow_id)
|
||||
if not workflow:
|
||||
return schemas.Response(success=False, message="工作流不存在")
|
||||
# 添加定时任务
|
||||
Scheduler().update_workflow_job(workflow)
|
||||
if not workflow.trigger_type or workflow.trigger_type == "timer":
|
||||
# 添加定时任务
|
||||
Scheduler().update_workflow_job(workflow)
|
||||
else:
|
||||
# 事件触发:添加到事件触发器
|
||||
WorkFlowManager().load_workflow_events(workflow_id)
|
||||
# 更新状态
|
||||
workflow.update_state(db, workflow_id, "W")
|
||||
return schemas.Response(success=True)
|
||||
@@ -140,11 +216,16 @@ def pause_workflow(workflow_id: int,
|
||||
"""
|
||||
停用工作流
|
||||
"""
|
||||
workflow = Workflow.get(db, workflow_id)
|
||||
workflow = WorkflowOper(db).get(workflow_id)
|
||||
if not workflow:
|
||||
return schemas.Response(success=False, message="工作流不存在")
|
||||
# 删除定时任务
|
||||
Scheduler().remove_workflow_job(workflow)
|
||||
# 根据触发类型进行不同处理
|
||||
if workflow.trigger_type == "timer":
|
||||
# 定时触发:移除定时任务
|
||||
Scheduler().remove_workflow_job(workflow)
|
||||
elif workflow.trigger_type == "event":
|
||||
# 事件触发:从事件触发器中移除
|
||||
WorkFlowManager().remove_workflow_event(workflow_id, workflow.event_type)
|
||||
# 停止工作流
|
||||
global_vars.stop_workflow(workflow_id)
|
||||
# 更新状态
|
||||
@@ -159,7 +240,8 @@ def reset_workflow(workflow_id: int,
|
||||
"""
|
||||
重置工作流
|
||||
"""
|
||||
workflow = Workflow.get(db, workflow_id)
|
||||
from app.db.workflow_oper import WorkflowOper
|
||||
workflow = WorkflowOper(db).get(workflow_id)
|
||||
if not workflow:
|
||||
return schemas.Response(success=False, message="工作流不存在")
|
||||
# 停止工作流
|
||||
@@ -169,3 +251,61 @@ def reset_workflow(workflow_id: int,
|
||||
# 删除缓存
|
||||
SystemConfigOper().delete(f"WorkflowCache-{workflow_id}")
|
||||
return schemas.Response(success=True)
|
||||
|
||||
|
||||
@router.get("/{workflow_id}", summary="工作流详情", response_model=schemas.Workflow)
|
||||
def get_workflow(workflow_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
_: schemas.TokenPayload = Depends(get_current_active_user)) -> Any:
|
||||
"""
|
||||
获取工作流详情
|
||||
"""
|
||||
from app.db.workflow_oper import WorkflowOper
|
||||
return WorkflowOper(db).get(workflow_id)
|
||||
|
||||
|
||||
@router.put("/{workflow_id}", summary="更新工作流", response_model=schemas.Response)
|
||||
def update_workflow(workflow: schemas.Workflow,
|
||||
db: Session = Depends(get_db),
|
||||
_: schemas.TokenPayload = Depends(get_current_active_user)) -> Any:
|
||||
"""
|
||||
更新工作流
|
||||
"""
|
||||
if not workflow.id:
|
||||
return schemas.Response(success=False, message="工作流ID不能为空")
|
||||
wf = WorkflowOper(db).get(workflow.id)
|
||||
if not wf:
|
||||
return schemas.Response(success=False, message="工作流不存在")
|
||||
if not wf.trigger_type:
|
||||
workflow.trigger_type = "timer"
|
||||
wf.update(db, workflow.dict())
|
||||
# 更新后的工作流对象
|
||||
updated_workflow = wf.get(workflow.id)
|
||||
# 更新定时任务
|
||||
Scheduler().update_workflow_job(updated_workflow)
|
||||
# 更新事件注册
|
||||
WorkFlowManager().update_workflow_event(updated_workflow)
|
||||
return schemas.Response(success=True, message="更新成功")
|
||||
|
||||
|
||||
@router.delete("/{workflow_id}", summary="删除工作流", response_model=schemas.Response)
|
||||
def delete_workflow(workflow_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
_: schemas.TokenPayload = Depends(get_current_active_user)) -> Any:
|
||||
"""
|
||||
删除工作流
|
||||
"""
|
||||
workflow = WorkflowOper(db).get(workflow_id)
|
||||
if not workflow:
|
||||
return schemas.Response(success=False, message="工作流不存在")
|
||||
if not workflow.trigger_type or workflow.trigger_type == "timer":
|
||||
# 定时触发:删除定时任务
|
||||
Scheduler().remove_workflow_job(workflow)
|
||||
else:
|
||||
# 事件触发:从事件触发器中移除
|
||||
WorkFlowManager().remove_workflow_event(workflow_id, workflow.event_type)
|
||||
# 删除工作流
|
||||
Workflow.delete(db, workflow_id)
|
||||
# 删除缓存
|
||||
SystemConfigOper().delete(f"WorkflowCache-{workflow_id}")
|
||||
return schemas.Response(success=True, message="删除成功")
|
||||
|
||||
@@ -188,6 +188,9 @@ class DownloadChain(ChainBase):
|
||||
f"Resource download canceled by event: {event_data.source},"
|
||||
f"Reason: {event_data.reason}")
|
||||
return None
|
||||
# 如果事件修改了下载路径,使用新路径
|
||||
if event_data.options and event_data.options.get("save_path"):
|
||||
save_path = event_data.options.get("save_path")
|
||||
|
||||
# 补充完整的media数据
|
||||
if not _media.genre_ids:
|
||||
|
||||
@@ -19,7 +19,6 @@ from app.utils.string import StringUtils
|
||||
|
||||
recognize_lock = Lock()
|
||||
scraping_lock = Lock()
|
||||
scraping_files = []
|
||||
|
||||
|
||||
class MediaChain(ChainBase):
|
||||
@@ -35,25 +34,25 @@ class MediaChain(ChainBase):
|
||||
switchs = SystemConfigOper().get(SystemConfigKey.ScrapingSwitchs) or {}
|
||||
# 默认配置
|
||||
default_switchs = {
|
||||
'movie_nfo': True, # 电影NFO
|
||||
'movie_poster': True, # 电影海报
|
||||
'movie_backdrop': True, # 电影背景图
|
||||
'movie_logo': True, # 电影Logo
|
||||
'movie_disc': True, # 电影光盘图
|
||||
'movie_banner': True, # 电影横幅图
|
||||
'movie_thumb': True, # 电影缩略图
|
||||
'tv_nfo': True, # 电视剧NFO
|
||||
'tv_poster': True, # 电视剧海报
|
||||
'tv_backdrop': True, # 电视剧背景图
|
||||
'tv_banner': True, # 电视剧横幅图
|
||||
'tv_logo': True, # 电视剧Logo
|
||||
'tv_thumb': True, # 电视剧缩略图
|
||||
'season_nfo': True, # 季NFO
|
||||
'season_poster': True, # 季海报
|
||||
'season_banner': True, # 季横幅图
|
||||
'season_thumb': True, # 季缩略图
|
||||
'episode_nfo': True, # 集NFO
|
||||
'episode_thumb': True # 集缩略图
|
||||
'movie_nfo': True, # 电影NFO
|
||||
'movie_poster': True, # 电影海报
|
||||
'movie_backdrop': True, # 电影背景图
|
||||
'movie_logo': True, # 电影Logo
|
||||
'movie_disc': True, # 电影光盘图
|
||||
'movie_banner': True, # 电影横幅图
|
||||
'movie_thumb': True, # 电影缩略图
|
||||
'tv_nfo': True, # 电视剧NFO
|
||||
'tv_poster': True, # 电视剧海报
|
||||
'tv_backdrop': True, # 电视剧背景图
|
||||
'tv_banner': True, # 电视剧横幅图
|
||||
'tv_logo': True, # 电视剧Logo
|
||||
'tv_thumb': True, # 电视剧缩略图
|
||||
'season_nfo': True, # 季NFO
|
||||
'season_poster': True, # 季海报
|
||||
'season_banner': True, # 季横幅图
|
||||
'season_thumb': True, # 季缩略图
|
||||
'episode_nfo': True, # 集NFO
|
||||
'episode_thumb': True # 集缩略图
|
||||
}
|
||||
# 合并用户配置和默认配置
|
||||
for key, default_value in default_switchs.items():
|
||||
@@ -344,23 +343,49 @@ class MediaChain(ChainBase):
|
||||
return
|
||||
event_data = event.event_data or {}
|
||||
fileitem: FileItem = event_data.get("fileitem")
|
||||
file_list: List[str] = event_data.get("file_list", [])
|
||||
meta: MetaBase = event_data.get("meta")
|
||||
mediainfo: MediaInfo = event_data.get("mediainfo")
|
||||
overwrite = event_data.get("overwrite", False)
|
||||
if not fileitem:
|
||||
return
|
||||
|
||||
# 刮削锁
|
||||
with scraping_lock:
|
||||
if fileitem.path in scraping_files:
|
||||
# 检查文件项是否存在
|
||||
storagechain = StorageChain()
|
||||
if not storagechain.get_item(fileitem):
|
||||
logger.warn(f"文件项不存在:{fileitem.path}")
|
||||
return
|
||||
scraping_files.append(fileitem.path)
|
||||
try:
|
||||
# 执行刮削
|
||||
self.scrape_metadata(fileitem=fileitem, meta=meta, mediainfo=mediainfo, overwrite=overwrite)
|
||||
finally:
|
||||
# 释放锁
|
||||
with scraping_lock:
|
||||
scraping_files.remove(fileitem.path)
|
||||
# 检查是否为目录
|
||||
if fileitem.type == "file":
|
||||
# 单个文件刮削
|
||||
self.scrape_metadata(fileitem=fileitem,
|
||||
mediainfo=mediainfo,
|
||||
init_folder=False,
|
||||
parent=storagechain.get_parent_item(fileitem),
|
||||
overwrite=overwrite)
|
||||
else:
|
||||
# 检查目的目录下是否已经有nfo刮削文件
|
||||
has_nfo_file = storagechain.any_files(fileitem, extensions=['.nfo'])
|
||||
if has_nfo_file and file_list:
|
||||
logger.info(f"目录 {fileitem.path} 已有NFO文件,开始增量刮削...")
|
||||
for file_path in file_list:
|
||||
file_item = storagechain.get_file_item(storage=fileitem.storage,
|
||||
path=Path(file_path))
|
||||
if file_item:
|
||||
# 对于电视剧文件,应该保存到与视频文件相同的目录
|
||||
# 而不是电视剧根目录
|
||||
self.scrape_metadata(fileitem=file_item,
|
||||
mediainfo=mediainfo,
|
||||
init_folder=False,
|
||||
parent=None, # 让函数内部自动获取正确的父目录
|
||||
overwrite=overwrite)
|
||||
else:
|
||||
# 执行全量刮削
|
||||
logger.info(f"开始全量刮削目录 {fileitem.path} ...")
|
||||
self.scrape_metadata(fileitem=fileitem, meta=meta, init_folder=True,
|
||||
mediainfo=mediainfo, overwrite=overwrite)
|
||||
|
||||
def scrape_metadata(self, fileitem: schemas.FileItem,
|
||||
meta: MetaBase = None, mediainfo: MediaInfo = None,
|
||||
@@ -407,8 +432,10 @@ class MediaChain(ChainBase):
|
||||
"""
|
||||
if not _fileitem or not _content or not _path:
|
||||
return
|
||||
# 保存文件到临时目录,文件名随机
|
||||
tmp_file = settings.TEMP_PATH / f"{_path.name}.{StringUtils.generate_random_str(10)}"
|
||||
# 保存文件到临时目录
|
||||
tmp_dir = settings.TEMP_PATH / StringUtils.generate_random_str(10)
|
||||
tmp_dir.mkdir(parents=True, exist_ok=True)
|
||||
tmp_file = tmp_dir / _path.name
|
||||
tmp_file.write_bytes(_content)
|
||||
# 获取文件的父目录
|
||||
try:
|
||||
@@ -427,7 +454,7 @@ class MediaChain(ChainBase):
|
||||
"""
|
||||
try:
|
||||
logger.info(f"正在下载图片:{_url} ...")
|
||||
r = RequestUtils(proxies=settings.PROXY, ua=settings.USER_AGENT).get_res(url=_url)
|
||||
r = RequestUtils(proxies=settings.PROXY, ua=settings.NORMAL_USER_AGENT).get_res(url=_url)
|
||||
if r:
|
||||
return r.content
|
||||
else:
|
||||
@@ -436,6 +463,9 @@ class MediaChain(ChainBase):
|
||||
logger.error(f"{_url} 图片下载失败:{str(err)}!")
|
||||
return None
|
||||
|
||||
if not fileitem:
|
||||
return
|
||||
|
||||
# 当前文件路径
|
||||
filepath = Path(fileitem.path)
|
||||
if fileitem.type == "file" \
|
||||
@@ -464,6 +494,8 @@ class MediaChain(ChainBase):
|
||||
movie_nfo = self.metadata_nfo(meta=meta, mediainfo=mediainfo)
|
||||
if movie_nfo:
|
||||
# 保存或上传nfo文件到上级目录
|
||||
if not parent:
|
||||
parent = storagechain.get_parent_item(fileitem)
|
||||
__save_file(_fileitem=parent, _path=nfo_path, _content=movie_nfo)
|
||||
else:
|
||||
logger.warn(f"{filepath.name} nfo文件生成失败!")
|
||||
@@ -494,8 +526,9 @@ class MediaChain(ChainBase):
|
||||
files = __list_files(_fileitem=fileitem)
|
||||
for file in files:
|
||||
self.scrape_metadata(fileitem=file,
|
||||
meta=meta, mediainfo=mediainfo,
|
||||
init_folder=False, parent=fileitem,
|
||||
mediainfo=mediainfo,
|
||||
init_folder=False,
|
||||
parent=fileitem,
|
||||
overwrite=overwrite)
|
||||
# 生成目录内图片文件
|
||||
if init_folder:
|
||||
@@ -587,11 +620,11 @@ class MediaChain(ChainBase):
|
||||
else:
|
||||
logger.info("集缩略图刮削已关闭,跳过")
|
||||
else:
|
||||
# 当前为目录,处理目录内的文件
|
||||
# 当前为电视剧目录,处理目录内的文件
|
||||
files = __list_files(_fileitem=fileitem)
|
||||
for file in files:
|
||||
self.scrape_metadata(fileitem=file,
|
||||
meta=meta, mediainfo=mediainfo,
|
||||
mediainfo=mediainfo,
|
||||
parent=fileitem if file.type == "file" else None,
|
||||
init_folder=True if file.type == "dir" else False,
|
||||
overwrite=overwrite)
|
||||
@@ -659,7 +692,8 @@ class MediaChain(ChainBase):
|
||||
# 只下载当前刮削季的图片
|
||||
image_season = "00" if "specials" in image_name else image_name[6:8]
|
||||
if image_season != str(season_meta.begin_season).rjust(2, '0'):
|
||||
logger.info(f"当前刮削季为:{season_meta.begin_season},跳过文件:{image_path}")
|
||||
logger.info(
|
||||
f"当前刮削季为:{season_meta.begin_season},跳过文件:{image_path}")
|
||||
continue
|
||||
if overwrite or not storagechain.get_file_item(storage=fileitem.storage,
|
||||
path=image_path):
|
||||
|
||||
@@ -130,7 +130,7 @@ class RecommendChain(ChainBase, metaclass=Singleton):
|
||||
# 请求远程图片
|
||||
referer = "https://movie.douban.com/" if "doubanio.com" in url else None
|
||||
proxies = settings.PROXY if not referer else None
|
||||
response = RequestUtils(ua=settings.USER_AGENT, proxies=proxies, referer=referer).get_res(url=url)
|
||||
response = RequestUtils(ua=settings.NORMAL_USER_AGENT, proxies=proxies, referer=referer).get_res(url=url)
|
||||
if not response:
|
||||
logger.debug(f"Empty response for URL: {url}")
|
||||
return
|
||||
|
||||
@@ -108,10 +108,6 @@ class SiteChain(ChainBase):
|
||||
"site_id": "*"
|
||||
})
|
||||
|
||||
# 如果不是大内存模式,进行垃圾回收
|
||||
if not settings.BIG_MEMORY_MODE:
|
||||
gc.collect()
|
||||
|
||||
return result
|
||||
|
||||
def is_special_site(self, domain: str) -> bool:
|
||||
@@ -271,16 +267,20 @@ class SiteChain(ChainBase):
|
||||
logger.error(f"获取站点页面失败:{url}")
|
||||
return favicon_url, None
|
||||
html = etree.HTML(html_text)
|
||||
if StringUtils.is_valid_html_element(html):
|
||||
fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href')
|
||||
if fav_link:
|
||||
favicon_url = urljoin(url, fav_link[0])
|
||||
try:
|
||||
if StringUtils.is_valid_html_element(html):
|
||||
fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href')
|
||||
if fav_link:
|
||||
favicon_url = urljoin(url, fav_link[0])
|
||||
|
||||
res = RequestUtils(cookies=cookie, timeout=15, ua=ua).get_res(url=favicon_url)
|
||||
if res:
|
||||
return favicon_url, base64.b64encode(res.content).decode()
|
||||
else:
|
||||
logger.error(f"获取站点图标失败:{favicon_url}")
|
||||
res = RequestUtils(cookies=cookie, timeout=15, ua=ua).get_res(url=favicon_url)
|
||||
if res:
|
||||
return favicon_url, base64.b64encode(res.content).decode()
|
||||
else:
|
||||
logger.error(f"获取站点图标失败:{favicon_url}")
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
return favicon_url, None
|
||||
|
||||
def sync_cookies(self, manual=False) -> Tuple[bool, str]:
|
||||
|
||||
@@ -178,15 +178,14 @@ class StorageChain(ChainBase):
|
||||
|
||||
if mtype:
|
||||
# 重命名格式
|
||||
rename_format = settings.TV_RENAME_FORMAT \
|
||||
if mtype == MediaType.TV else settings.MOVIE_RENAME_FORMAT
|
||||
# 计算重命名中的文件夹层数
|
||||
rename_format_level = len(rename_format.split("/")) - 1
|
||||
if rename_format_level < 1:
|
||||
rename_format = settings.RENAME_FORMAT(mtype)
|
||||
media_path = DirectoryHelper.get_media_root_path(
|
||||
rename_format, rename_path=Path(fileitem.path)
|
||||
)
|
||||
if not media_path:
|
||||
return True
|
||||
# 处理媒体文件根目录
|
||||
dir_item = self.get_file_item(storage=fileitem.storage,
|
||||
path=Path(fileitem.path).parents[rename_format_level - 1])
|
||||
dir_item = self.get_file_item(storage=fileitem.storage, path=media_path)
|
||||
else:
|
||||
# 处理上级目录
|
||||
dir_item = self.get_parent_item(fileitem)
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import copy
|
||||
import gc
|
||||
import json
|
||||
import random
|
||||
import threading
|
||||
@@ -39,6 +38,8 @@ class SubscribeChain(ChainBase):
|
||||
"""
|
||||
|
||||
_rlock = threading.RLock()
|
||||
# 避免莫名原因导致长时间持有锁
|
||||
_LOCK_TIMOUT = 3600 * 2
|
||||
|
||||
def add(self, title: str, year: str,
|
||||
mtype: MediaType = None,
|
||||
@@ -279,8 +280,15 @@ class SubscribeChain(ChainBase):
|
||||
:param manual: 是否手动搜索
|
||||
:return: 更新订阅状态为R或删除订阅
|
||||
"""
|
||||
with self._rlock:
|
||||
logger.debug(f"search lock acquired at {datetime.now()}")
|
||||
lock_acquired = False
|
||||
try:
|
||||
if lock_acquired := self._rlock.acquire(
|
||||
blocking=True, timeout=self._LOCK_TIMOUT
|
||||
):
|
||||
logger.debug(f"search lock acquired at {datetime.now()}")
|
||||
else:
|
||||
logger.warn("search上锁超时")
|
||||
|
||||
subscribeoper = SubscribeOper()
|
||||
if sid:
|
||||
subscribe = subscribeoper.get(sid)
|
||||
@@ -434,14 +442,13 @@ class SubscribeChain(ChainBase):
|
||||
else:
|
||||
self.messagehelper.put('没有找到订阅!', title="订阅搜索", role="system")
|
||||
|
||||
logger.debug(f"search Lock released at {datetime.now()}")
|
||||
finally:
|
||||
subscribes.clear()
|
||||
del subscribes
|
||||
|
||||
# 如果不是大内存模式,进行垃圾回收
|
||||
if not settings.BIG_MEMORY_MODE:
|
||||
gc.collect()
|
||||
finally:
|
||||
if lock_acquired:
|
||||
self._rlock.release()
|
||||
logger.debug(f"search Lock released at {datetime.now()}")
|
||||
|
||||
def update_subscribe_priority(self, subscribe: Subscribe, meta: MetaBase,
|
||||
mediainfo: MediaInfo, downloads: Optional[List[Context]]):
|
||||
@@ -514,9 +521,6 @@ class SubscribeChain(ChainBase):
|
||||
self.match(
|
||||
TorrentsChain().refresh(sites=sites)
|
||||
)
|
||||
# 如果不是大内存模式,进行垃圾回收
|
||||
if not settings.BIG_MEMORY_MODE:
|
||||
gc.collect()
|
||||
|
||||
@staticmethod
|
||||
def get_sub_sites(subscribe: Subscribe) -> List[int]:
|
||||
@@ -546,10 +550,15 @@ class SubscribeChain(ChainBase):
|
||||
:return: 返回[]代表所有站点命中,返回None代表没有订阅
|
||||
"""
|
||||
ret_sites = []
|
||||
subscribes = SubscribeOper().list()
|
||||
if not subscribes:
|
||||
# 没有订阅
|
||||
return None
|
||||
# 刷新订阅选中的Rss站点
|
||||
for subscribe in SubscribeOper().list(self.get_states_for_search('R')):
|
||||
for subscribe in subscribes:
|
||||
# 刷新选中的站点
|
||||
ret_sites.extend(self.get_sub_sites(subscribe))
|
||||
if subscribe.state in self.get_states_for_search('R'):
|
||||
ret_sites.extend(self.get_sub_sites(subscribe))
|
||||
# 去重
|
||||
if ret_sites:
|
||||
ret_sites = list(set(ret_sites))
|
||||
@@ -564,8 +573,14 @@ class SubscribeChain(ChainBase):
|
||||
logger.warn('没有缓存资源,无法匹配订阅')
|
||||
return
|
||||
|
||||
with self._rlock:
|
||||
logger.debug(f"match lock acquired at {datetime.now()}")
|
||||
lock_acquired = False
|
||||
try:
|
||||
if lock_acquired := self._rlock.acquire(
|
||||
blocking=True, timeout=self._LOCK_TIMOUT
|
||||
):
|
||||
logger.debug(f"match lock acquired at {datetime.now()}")
|
||||
else:
|
||||
logger.warn("match上锁超时")
|
||||
|
||||
# 预识别所有未识别的种子
|
||||
processed_torrents: Dict[str, List[Context]] = {}
|
||||
@@ -806,7 +821,8 @@ class SubscribeChain(ChainBase):
|
||||
username=subscribe.username,
|
||||
save_path=subscribe.save_path,
|
||||
downloader=subscribe.downloader,
|
||||
source=self.get_subscribe_source_keyword(subscribe)
|
||||
source=self.get_subscribe_source_keyword(
|
||||
subscribe)
|
||||
)
|
||||
|
||||
# 同步外部修改,更新订阅信息
|
||||
@@ -821,8 +837,10 @@ class SubscribeChain(ChainBase):
|
||||
del processed_torrents
|
||||
subscribes.clear()
|
||||
del subscribes
|
||||
|
||||
logger.debug(f"match Lock released at {datetime.now()}")
|
||||
finally:
|
||||
if lock_acquired:
|
||||
self._rlock.release()
|
||||
logger.debug(f"match Lock released at {datetime.now()}")
|
||||
|
||||
def check(self):
|
||||
"""
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Union, Optional
|
||||
|
||||
from app.chain import ChainBase
|
||||
from app.core.config import settings
|
||||
from app.core.plugin import PluginManager
|
||||
from app.log import logger
|
||||
from app.schemas import Notification, MessageChannel
|
||||
from app.utils.http import RequestUtils
|
||||
@@ -136,13 +137,6 @@ class SystemChain(ChainBase):
|
||||
shutil.rmtree(target_path)
|
||||
shutil.copytree(item, target_path)
|
||||
logger.info(f"已恢复插件目录: {item.name}")
|
||||
# 安装依赖
|
||||
requirements_file = target_path / "requirements.txt"
|
||||
if requirements_file.exists():
|
||||
logger.info(f"正在安装插件 {item.name} 的依赖...")
|
||||
success, message = PluginHelper.pip_install_with_fallback(requirements_file)
|
||||
if not success:
|
||||
logger.warn(f"插件 {item.name} 依赖安装失败: {message}")
|
||||
restored_count += 1
|
||||
# 如果是文件
|
||||
elif item.is_file():
|
||||
@@ -155,6 +149,9 @@ class SystemChain(ChainBase):
|
||||
|
||||
logger.info(f"插件恢复完成,共恢复 {restored_count} 个项目")
|
||||
|
||||
# 安装缺少的依赖
|
||||
PluginManager.install_plugin_missing_dependencies()
|
||||
|
||||
# 删除备份目录
|
||||
try:
|
||||
shutil.rmtree(backup_dir)
|
||||
|
||||
@@ -140,6 +140,16 @@ class TorrentsChain(ChainBase):
|
||||
:param stype: 强制指定缓存类型,spider:爬虫缓存,rss:rss缓存
|
||||
:param sites: 强制指定站点ID列表,为空则读取设置的订阅站点
|
||||
"""
|
||||
|
||||
def __is_no_cache_site(_domain: str) -> bool:
|
||||
"""
|
||||
判断站点是否不需要缓存
|
||||
"""
|
||||
for url_key in settings.NO_CACHE_SITE_KEY.split(','):
|
||||
if url_key in _domain:
|
||||
return True
|
||||
return False
|
||||
|
||||
# 刷新类型
|
||||
if not stype:
|
||||
stype = settings.SUBSCRIBE_MODE
|
||||
@@ -169,7 +179,15 @@ class TorrentsChain(ChainBase):
|
||||
domains.append(domain)
|
||||
if stype == "spider":
|
||||
# 刷新首页种子
|
||||
torrents: List[TorrentInfo] = self.browse(domain=domain)
|
||||
torrents: List[TorrentInfo] = []
|
||||
# 读取第0页和第1页
|
||||
for page in range(2):
|
||||
page_torrents = self.browse(domain=domain, page=page)
|
||||
if page_torrents:
|
||||
torrents.extend(page_torrents)
|
||||
else:
|
||||
# 如果某一页没有数据,说明已经到最后一页,停止获取
|
||||
break
|
||||
else:
|
||||
# 刷新RSS种子
|
||||
torrents: List[TorrentInfo] = self.rss(domain=domain)
|
||||
@@ -178,11 +196,16 @@ class TorrentsChain(ChainBase):
|
||||
# 取前N条
|
||||
torrents = torrents[:settings.CONF.refresh]
|
||||
if torrents:
|
||||
# 过滤出没有处理过的种子 - 优化:使用集合查找,避免重复创建字符串列表
|
||||
cached_signatures = {f'{t.torrent_info.title}{t.torrent_info.description}'
|
||||
for t in torrents_cache.get(domain) or []}
|
||||
torrents = [torrent for torrent in torrents
|
||||
if f'{torrent.title}{torrent.description}' not in cached_signatures]
|
||||
if __is_no_cache_site(domain):
|
||||
# 不需要缓存的站点,直接处理
|
||||
logger.info(f'{indexer.get("name")} 有 {len(torrents)} 个种子 (不缓存)')
|
||||
torrents_cache[domain] = []
|
||||
else:
|
||||
# 过滤出没有处理过的种子 - 优化:使用集合查找,避免重复创建字符串列表
|
||||
cached_signatures = {f'{t.torrent_info.title}{t.torrent_info.description}'
|
||||
for t in torrents_cache.get(domain) or []}
|
||||
torrents = [torrent for torrent in torrents
|
||||
if f'{torrent.title}{torrent.description}' not in cached_signatures]
|
||||
if torrents:
|
||||
logger.info(f'{indexer.get("name")} 有 {len(torrents)} 个新种子')
|
||||
else:
|
||||
|
||||
@@ -5,7 +5,6 @@ import threading
|
||||
import traceback
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from queue import Queue
|
||||
from time import sleep
|
||||
from typing import List, Optional, Tuple, Union, Dict, Callable
|
||||
|
||||
@@ -16,9 +15,9 @@ from app.chain.storage import StorageChain
|
||||
from app.chain.tmdb import TmdbChain
|
||||
from app.core.config import settings, global_vars
|
||||
from app.core.context import MediaInfo
|
||||
from app.core.event import eventmanager
|
||||
from app.core.meta import MetaBase
|
||||
from app.core.metainfo import MetaInfoPath
|
||||
from app.core.event import eventmanager
|
||||
from app.db.downloadhistory_oper import DownloadHistoryOper
|
||||
from app.db.models.downloadhistory import DownloadHistory
|
||||
from app.db.models.transferhistory import TransferHistory
|
||||
@@ -28,11 +27,11 @@ from app.helper.directory import DirectoryHelper
|
||||
from app.helper.format import FormatParser
|
||||
from app.helper.progress import ProgressHelper
|
||||
from app.log import logger
|
||||
from app.schemas import StorageOperSelectionEventData
|
||||
from app.schemas import TransferInfo, TransferTorrent, Notification, EpisodeFormat, FileItem, TransferDirectoryConf, \
|
||||
TransferTask, TransferQueue, TransferJob, TransferJobTask
|
||||
from app.schemas.types import TorrentStatus, EventType, MediaType, ProgressKey, NotificationType, MessageChannel, \
|
||||
SystemConfigKey, ChainEventType, ContentType
|
||||
from app.schemas import StorageOperSelectionEventData
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.string import StringUtils
|
||||
|
||||
@@ -213,6 +212,7 @@ class JobManager:
|
||||
set(self._season_episodes[mediaid]) - set(task.meta.episode_list)
|
||||
)
|
||||
return task
|
||||
return None
|
||||
|
||||
def remove_job(self, task: TransferTask) -> Optional[TransferJob]:
|
||||
"""
|
||||
@@ -226,6 +226,7 @@ class JobManager:
|
||||
if __mediaid__ in self._season_episodes:
|
||||
self._season_episodes.pop(__mediaid__)
|
||||
return self._job_view.pop(__mediaid__)
|
||||
return None
|
||||
|
||||
def is_done(self, task: TransferTask) -> bool:
|
||||
"""
|
||||
@@ -311,7 +312,7 @@ class JobManager:
|
||||
|
||||
def count(self, media: MediaInfo, season: Optional[int] = None) -> int:
|
||||
"""
|
||||
获取某项任务总数
|
||||
获取某项任务成功总数
|
||||
"""
|
||||
__mediaid__ = self.__get_media_id(media=media, season=season)
|
||||
with job_lock:
|
||||
@@ -322,7 +323,7 @@ class JobManager:
|
||||
|
||||
def size(self, media: MediaInfo, season: Optional[int] = None) -> int:
|
||||
"""
|
||||
获取某项任务总大小
|
||||
获取某项任务成功文件总大小
|
||||
"""
|
||||
__mediaid__ = self.__get_media_id(media=media, season=season)
|
||||
with job_lock:
|
||||
@@ -359,22 +360,20 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
文件整理处理链
|
||||
"""
|
||||
|
||||
# 可处理的文件后缀
|
||||
all_exts = settings.RMT_MEDIAEXT
|
||||
|
||||
# 待整理任务队列
|
||||
_queue = Queue()
|
||||
|
||||
# 文件整理线程
|
||||
_transfer_thread = None
|
||||
|
||||
# 队列间隔时间(秒)
|
||||
_transfer_interval = 15
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
# 可处理的文件后缀
|
||||
self.all_exts = settings.RMT_MEDIAEXT
|
||||
# 待整理任务队列
|
||||
self._queue = queue.Queue()
|
||||
# 文件整理线程
|
||||
self._transfer_thread = None
|
||||
# 队列间隔时间(秒)
|
||||
self._transfer_interval = 15
|
||||
# 事件管理器
|
||||
self.jobview = JobManager()
|
||||
|
||||
# 车移成功的文件清单
|
||||
self._success_target_files: Dict[str, List[str]] = {}
|
||||
# 启动整理任务
|
||||
self.__init()
|
||||
|
||||
@@ -391,6 +390,44 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
"""
|
||||
整理完成后处理
|
||||
"""
|
||||
|
||||
def __do_finished():
|
||||
"""
|
||||
完成时发送消息、刮削事件、移除任务等
|
||||
"""
|
||||
# 更新文件数量
|
||||
transferinfo.file_count = self.jobview.count(task.mediainfo, task.meta.begin_season) or 1
|
||||
# 更新文件大小
|
||||
transferinfo.total_size = self.jobview.size(task.mediainfo,
|
||||
task.meta.begin_season) or task.fileitem.size
|
||||
# 更新文件清单
|
||||
transferinfo.file_list_new = self._success_target_files.pop(transferinfo.target_diritem.path, [])
|
||||
# 发送通知,实时手动整理时不发
|
||||
if transferinfo.need_notify and (task.background or not task.manual):
|
||||
se_str = None
|
||||
if task.mediainfo.type == MediaType.TV:
|
||||
season_episodes = self.jobview.season_episodes(task.mediainfo, task.meta.begin_season)
|
||||
if season_episodes:
|
||||
se_str = f"{task.meta.season} {StringUtils.format_ep(season_episodes)}"
|
||||
else:
|
||||
se_str = f"{task.meta.season}"
|
||||
self.send_transfer_message(meta=task.meta,
|
||||
mediainfo=task.mediainfo,
|
||||
transferinfo=transferinfo,
|
||||
season_episode=se_str,
|
||||
username=task.username)
|
||||
# 刮削事件
|
||||
if transferinfo.need_scrape:
|
||||
self.eventmanager.send_event(EventType.MetadataScrape, {
|
||||
'meta': task.meta,
|
||||
'mediainfo': task.mediainfo,
|
||||
'fileitem': transferinfo.target_diritem,
|
||||
'file_list': transferinfo.file_list_new,
|
||||
'overwrite': False
|
||||
})
|
||||
# 移除已完成的任务
|
||||
self.jobview.remove_job(task)
|
||||
|
||||
transferhis = TransferHistoryOper()
|
||||
if not transferinfo.success:
|
||||
# 转移失败
|
||||
@@ -416,6 +453,10 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
))
|
||||
# 整理失败
|
||||
self.jobview.fail_task(task)
|
||||
with task_lock:
|
||||
# 整理完成且有成功的任务时
|
||||
if self.jobview.is_finished(task):
|
||||
__do_finished()
|
||||
return False, transferinfo.message
|
||||
|
||||
# 转移成功
|
||||
@@ -444,55 +485,57 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
})
|
||||
|
||||
with task_lock:
|
||||
# 登记转移成功文件清单
|
||||
target_dir_path = transferinfo.target_diritem.path
|
||||
target_files = transferinfo.file_list_new
|
||||
if self._success_target_files.get(target_dir_path):
|
||||
self._success_target_files[target_dir_path].extend(target_files)
|
||||
else:
|
||||
self._success_target_files[target_dir_path] = target_files
|
||||
# 全部整理成功时
|
||||
if self.jobview.is_success(task):
|
||||
# 移动模式删除空目录
|
||||
if transferinfo.transfer_type in ["move"]:
|
||||
# 所有成功的业务
|
||||
tasks = self.jobview.success_tasks(task.mediainfo, task.meta.begin_season)
|
||||
# 记录已处理的种子hash
|
||||
processed_hashes = set()
|
||||
storagechain = StorageChain()
|
||||
downloadhistoryoper = DownloadHistoryOper()
|
||||
for t in tasks:
|
||||
# 下载器hash
|
||||
if t.download_hash and t.download_hash not in processed_hashes:
|
||||
processed_hashes.add(t.download_hash)
|
||||
if self.remove_torrents(t.download_hash, downloader=t.downloader):
|
||||
logger.info(f"移动模式删除种子成功:{t.download_hash} ")
|
||||
# 删除残留目录
|
||||
if t.fileitem:
|
||||
storagechain.delete_media_file(t.fileitem, delete_self=False)
|
||||
if not t.download_hash:
|
||||
continue
|
||||
# 获取种子保存目录
|
||||
seed_dir_path = self.__get_torrent_save_path(download_hash=t.download_hash,
|
||||
downloader=t.downloader)
|
||||
if not seed_dir_path:
|
||||
# 如果无法从下载器获取,则尝试从历史记录获取
|
||||
download_history = downloadhistoryoper.get_by_hash(t.download_hash)
|
||||
if download_history and download_history.path:
|
||||
seed_dir_path = download_history.path
|
||||
else:
|
||||
logger.warn(f"无法获取种子 {t.download_hash} 的保存路径")
|
||||
continue
|
||||
|
||||
# 检查种子目录下是否还有有效媒体文件
|
||||
seed_dir_item = storagechain.get_file_item(storage=t.fileitem.storage,
|
||||
path=Path(seed_dir_path))
|
||||
if seed_dir_item and seed_dir_item.type == "dir":
|
||||
remain_files = storagechain.list_files(seed_dir_item, recursion=True)
|
||||
has_media = any(
|
||||
f.extension and f.extension.lower() in [ext.lstrip('.') for ext in self.all_exts]
|
||||
for f in remain_files if f.type == "file"
|
||||
)
|
||||
if not has_media:
|
||||
if self.remove_torrents(t.download_hash, downloader=t.downloader):
|
||||
logger.info(f"移动模式删除种子成功:{t.download_hash} ")
|
||||
# 删除残留目录
|
||||
storagechain.delete_media_file(seed_dir_item, delete_self=False)
|
||||
else:
|
||||
logger.info(
|
||||
f"种子目录 {seed_dir_path} 还有未整理的媒体文件,暂不删除种子和残留目录")
|
||||
# 整理完成且有成功的任务时
|
||||
if self.jobview.is_finished(task):
|
||||
# 发送通知,实时手动整理时不发
|
||||
if transferinfo.need_notify and (task.background or not task.manual):
|
||||
se_str = None
|
||||
if task.mediainfo.type == MediaType.TV:
|
||||
season_episodes = self.jobview.season_episodes(task.mediainfo, task.meta.begin_season)
|
||||
if season_episodes:
|
||||
se_str = f"{task.meta.season} {StringUtils.format_ep(season_episodes)}"
|
||||
else:
|
||||
se_str = f"{task.meta.season}"
|
||||
# 更新文件数量
|
||||
transferinfo.file_count = self.jobview.count(task.mediainfo, task.meta.begin_season) or 1
|
||||
# 更新文件大小
|
||||
transferinfo.total_size = self.jobview.size(task.mediainfo,
|
||||
task.meta.begin_season) or task.fileitem.size
|
||||
self.send_transfer_message(meta=task.meta,
|
||||
mediainfo=task.mediainfo,
|
||||
transferinfo=transferinfo,
|
||||
season_episode=se_str,
|
||||
username=task.username)
|
||||
# 刮削事件
|
||||
if transferinfo.need_scrape:
|
||||
self.eventmanager.send_event(EventType.MetadataScrape, {
|
||||
'meta': task.meta,
|
||||
'mediainfo': task.mediainfo,
|
||||
'fileitem': transferinfo.target_diritem
|
||||
})
|
||||
|
||||
# 移除已完成的任务
|
||||
self.jobview.remove_job(task)
|
||||
__do_finished()
|
||||
|
||||
return True, ""
|
||||
|
||||
@@ -847,7 +890,7 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
state, errmsg = self.do_transfer(
|
||||
fileitem=FileItem(
|
||||
storage="local",
|
||||
path=str(file_path).replace("\\", "/"),
|
||||
path=file_path.as_posix(),
|
||||
type="dir" if not file_path.is_file() else "file",
|
||||
name=file_path.name,
|
||||
size=file_path.stat().st_size,
|
||||
@@ -867,10 +910,6 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
torrents.clear()
|
||||
del torrents
|
||||
|
||||
# 如果不是大内存模式,进行垃圾回收
|
||||
if not settings.BIG_MEMORY_MODE:
|
||||
gc.collect()
|
||||
|
||||
# 结束
|
||||
logger.info("所有下载器中下载完成的文件已整理完成")
|
||||
return True
|
||||
@@ -1099,7 +1138,8 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
# 自定义识别
|
||||
if formaterHandler:
|
||||
# 开始集、结束集、PART
|
||||
begin_ep, end_ep, part = formaterHandler.split_episode(file_name=file_path.name, file_meta=file_meta)
|
||||
begin_ep, end_ep, part = formaterHandler.split_episode(file_name=file_path.name,
|
||||
file_meta=file_meta)
|
||||
if begin_ep is not None:
|
||||
file_meta.begin_episode = begin_ep
|
||||
file_meta.part = part
|
||||
@@ -1204,7 +1244,8 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
key=ProgressKey.FileTransfer)
|
||||
progress.end(ProgressKey.FileTransfer)
|
||||
|
||||
return all_success, ",".join(err_msgs)
|
||||
error_msg = "、".join(err_msgs[:2]) + (f",等{len(err_msgs)}个文件错误!" if len(err_msgs) > 2 else "")
|
||||
return all_success, error_msg
|
||||
|
||||
def remote_transfer(self, arg_str: str, channel: MessageChannel,
|
||||
userid: Union[str, int] = None, source: Optional[str] = None):
|
||||
@@ -1341,7 +1382,8 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
mediainfo: MediaInfo = MediaChain().recognize_media(tmdbid=tmdbid, doubanid=doubanid,
|
||||
mtype=mtype, episode_group=episode_group)
|
||||
if not mediainfo:
|
||||
return False, f"媒体信息识别失败,tmdbid:{tmdbid},doubanid:{doubanid},type: {mtype.value}"
|
||||
return (False,
|
||||
f"媒体信息识别失败,tmdbid:{tmdbid},doubanid:{doubanid},type: {mtype.value if mtype else None}")
|
||||
else:
|
||||
# 更新媒体图片
|
||||
self.obtain_images(mediainfo=mediainfo)
|
||||
@@ -1411,3 +1453,20 @@ class TransferChain(ChainBase, metaclass=Singleton):
|
||||
season_episode=season_episode,
|
||||
username=username
|
||||
)
|
||||
|
||||
def __get_torrent_save_path(self, download_hash: str, downloader: str) -> Optional[str]:
|
||||
"""
|
||||
从下载器获取种子的保存路径
|
||||
:param download_hash: 种子Hash
|
||||
:param downloader: 下载器名称
|
||||
:return: 种子保存路径,如果获取失败返回None
|
||||
"""
|
||||
try:
|
||||
# 通过下载器获取种子信息
|
||||
torrents = self.list_torrents(hashs=download_hash, downloader=downloader)
|
||||
if not torrents:
|
||||
return None
|
||||
return torrents[0].path
|
||||
except Exception as e:
|
||||
logger.error(f"获取种子 {download_hash} 保存路径失败:{e}")
|
||||
return None
|
||||
|
||||
@@ -10,11 +10,13 @@ from pydantic.fields import Callable
|
||||
|
||||
from app.chain import ChainBase
|
||||
from app.core.config import global_vars
|
||||
from app.core.event import Event, eventmanager
|
||||
from app.core.workflow import WorkFlowManager
|
||||
from app.db.models import Workflow
|
||||
from app.db.workflow_oper import WorkflowOper
|
||||
from app.log import logger
|
||||
from app.schemas import ActionContext, ActionFlow, Action, ActionExecution
|
||||
from app.schemas.types import EventType
|
||||
|
||||
|
||||
class WorkflowExecutor:
|
||||
@@ -188,6 +190,16 @@ class WorkflowChain(ChainBase):
|
||||
工作流链
|
||||
"""
|
||||
|
||||
@eventmanager.register(EventType.WorkflowExecute)
|
||||
def event_process(self, event: Event):
|
||||
"""
|
||||
事件触发工作流执行
|
||||
"""
|
||||
workflow_id = event.event_data.get('workflow_id')
|
||||
if not workflow_id:
|
||||
return
|
||||
self.process(workflow_id, from_begin=False)
|
||||
|
||||
@staticmethod
|
||||
def process(workflow_id: int, from_begin: Optional[bool] = True) -> Tuple[bool, str]:
|
||||
"""
|
||||
@@ -225,7 +237,7 @@ class WorkflowChain(ChainBase):
|
||||
logger.warn(f"工作流 {workflow.name} 无流程")
|
||||
return False, "工作流无流程"
|
||||
|
||||
logger.info(f"开始处理 {workflow.name},共 {len(workflow.actions)} 个动作 ...")
|
||||
logger.info(f"开始执行工作流 {workflow.name},共 {len(workflow.actions)} 个动作 ...")
|
||||
workflowoper.start(workflow_id)
|
||||
|
||||
# 执行工作流
|
||||
@@ -247,3 +259,17 @@ class WorkflowChain(ChainBase):
|
||||
获取工作流列表
|
||||
"""
|
||||
return WorkflowOper().list_enabled()
|
||||
|
||||
@staticmethod
|
||||
def get_timer_workflows() -> List[Workflow]:
|
||||
"""
|
||||
获取定时触发的工作流列表
|
||||
"""
|
||||
return WorkflowOper().get_timer_triggered_workflows()
|
||||
|
||||
@staticmethod
|
||||
def get_event_workflows() -> List[Workflow]:
|
||||
"""
|
||||
获取事件触发的工作流列表
|
||||
"""
|
||||
return WorkflowOper().get_event_triggered_workflows()
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import secrets
|
||||
import sys
|
||||
import threading
|
||||
@@ -11,8 +13,10 @@ from dotenv import set_key
|
||||
from pydantic import BaseModel, BaseSettings, validator, Field
|
||||
|
||||
from app.log import logger, log_settings, LogConfigModel
|
||||
from app.schemas import MediaType
|
||||
from app.utils.system import SystemUtils
|
||||
from app.utils.url import UrlUtils
|
||||
from version import APP_VERSION
|
||||
|
||||
|
||||
class SystemConfModel(BaseModel):
|
||||
@@ -211,6 +215,8 @@ class ConfigModel(BaseModel):
|
||||
SITEDATA_REFRESH_INTERVAL: int = 6
|
||||
# 读取和发送站点消息
|
||||
SITE_MESSAGE: bool = True
|
||||
# 不能缓存站点资源的站点域名,多个使用,分隔
|
||||
NO_CACHE_SITE_KEY: str = "m-team"
|
||||
# 种子标签
|
||||
TORRENT_TAG: str = "MOVIEPILOT"
|
||||
# 下载站点字幕
|
||||
@@ -229,8 +235,6 @@ class ConfigModel(BaseModel):
|
||||
COOKIECLOUD_INTERVAL: Optional[int] = 60 * 24
|
||||
# CookieCloud同步黑名单,多个域名,分割
|
||||
COOKIECLOUD_BLACKLIST: Optional[str] = None
|
||||
# CookieCloud对应的浏览器UA
|
||||
USER_AGENT: str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.57"
|
||||
# 电影重命名格式
|
||||
MOVIE_RENAME_FORMAT: str = "{{title}}{% if year %} ({{year}}){% endif %}" \
|
||||
"/{{title}}{% if year %} ({{year}}){% endif %}{% if part %}-{{part}}{% endif %}{% if videoFormat %} - {{videoFormat}}{% endif %}" \
|
||||
@@ -274,12 +278,6 @@ class ConfigModel(BaseModel):
|
||||
REPO_GITHUB_TOKEN: Optional[str] = None
|
||||
# 大内存模式
|
||||
BIG_MEMORY_MODE: bool = False
|
||||
# 是否启用内存监控
|
||||
MEMORY_ANALYSIS: bool = False
|
||||
# 内存快照间隔(分钟)
|
||||
MEMORY_SNAPSHOT_INTERVAL: int = 30
|
||||
# 保留的内存快照文件数量
|
||||
MEMORY_SNAPSHOT_KEEP_COUNT: int = 20
|
||||
# 全局图片缓存,将媒体图片缓存到本地
|
||||
GLOBAL_IMAGE_CACHE: bool = False
|
||||
# 是否启用编码探测的性能模式
|
||||
@@ -311,6 +309,12 @@ class ConfigModel(BaseModel):
|
||||
DEFAULT_SUB: Optional[str] = "zh-cn"
|
||||
# Docker Client API地址
|
||||
DOCKER_CLIENT_API: Optional[str] = "tcp://127.0.0.1:38379"
|
||||
# 工作流数据共享
|
||||
WORKFLOW_STATISTIC_SHARE: bool = True
|
||||
# 对rclone进行快照对比时,是否检查文件夹的修改时间
|
||||
RCLONE_SNAPSHOT_CHECK_FOLDER_MODTIME = True
|
||||
# 对OpenList进行快照对比时,是否检查文件夹的修改时间
|
||||
OPENLIST_SNAPSHOT_CHECK_FOLDER_MODTIME = True
|
||||
|
||||
|
||||
class Settings(BaseSettings, ConfigModel, LogConfigModel):
|
||||
@@ -510,6 +514,20 @@ class Settings(BaseSettings, ConfigModel, LogConfigModel):
|
||||
"""
|
||||
return "v2"
|
||||
|
||||
@property
|
||||
def USER_AGENT(self) -> str:
|
||||
"""
|
||||
全局用户代理字符串
|
||||
"""
|
||||
return f"{self.PROJECT_NAME}/{APP_VERSION[1:]} ({platform.system()} {platform.release()}; {SystemUtils.cpu_arch()})"
|
||||
|
||||
@property
|
||||
def NORMAL_USER_AGENT(self) -> str:
|
||||
"""
|
||||
默认浏览器用户代理字符串
|
||||
"""
|
||||
return "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"
|
||||
|
||||
@property
|
||||
def INNER_CONFIG_PATH(self):
|
||||
return self.ROOT_PATH / "config"
|
||||
@@ -606,7 +624,7 @@ class Settings(BaseSettings, ConfigModel, LogConfigModel):
|
||||
if self.GITHUB_TOKEN:
|
||||
return {
|
||||
"Authorization": f"Bearer {self.GITHUB_TOKEN}",
|
||||
"User-Agent": self.USER_AGENT,
|
||||
"User-Agent": self.NORMAL_USER_AGENT,
|
||||
}
|
||||
return {}
|
||||
|
||||
@@ -635,7 +653,7 @@ class Settings(BaseSettings, ConfigModel, LogConfigModel):
|
||||
continue
|
||||
headers[repo_info] = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"User-Agent": self.USER_AGENT,
|
||||
"User-Agent": self.NORMAL_USER_AGENT,
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"处理令牌对 '{token_pair}' 时出错: {e}")
|
||||
@@ -655,6 +673,23 @@ class Settings(BaseSettings, ConfigModel, LogConfigModel):
|
||||
return None
|
||||
return UrlUtils.combine_url(host=self.APP_DOMAIN, path=url)
|
||||
|
||||
def RENAME_FORMAT(self, media_type: MediaType):
|
||||
"""
|
||||
获取指定类型的重命名格式
|
||||
|
||||
:param media_type: MediaType.TV 或 MediaType.Movie
|
||||
:return: 重命名格式
|
||||
"""
|
||||
rename_format = (
|
||||
self.TV_RENAME_FORMAT
|
||||
if media_type == MediaType.TV
|
||||
else self.MOVIE_RENAME_FORMAT
|
||||
)
|
||||
# 规范重命名格式
|
||||
rename_format = rename_format.replace("\\", "/")
|
||||
rename_format = re.sub(r'/+', '/', rename_format)
|
||||
return rename_format.strip("/")
|
||||
|
||||
|
||||
# 实例化配置
|
||||
settings = Settings()
|
||||
|
||||
@@ -193,7 +193,7 @@ class MediaInfo:
|
||||
# LOGO
|
||||
logo_path: str = None
|
||||
# 评分
|
||||
vote_average: float = 0.0
|
||||
vote_average: float = None
|
||||
# 描述
|
||||
overview: str = None
|
||||
# 风格ID
|
||||
@@ -237,9 +237,9 @@ class MediaInfo:
|
||||
# 流媒体平台
|
||||
networks: list = field(default_factory=list)
|
||||
# 集数
|
||||
number_of_episodes: int = 0
|
||||
number_of_episodes: int = None
|
||||
# 季数
|
||||
number_of_seasons: int = 0
|
||||
number_of_seasons: int = None
|
||||
# 原产国
|
||||
origin_country: list = field(default_factory=list)
|
||||
# 原名
|
||||
@@ -255,9 +255,9 @@ class MediaInfo:
|
||||
# 标签
|
||||
tagline: str = None
|
||||
# 评价数量
|
||||
vote_count: int = 0
|
||||
vote_count: int = None
|
||||
# 流行度
|
||||
popularity: int = 0
|
||||
popularity: int = None
|
||||
# 时长
|
||||
runtime: int = None
|
||||
# 下一集
|
||||
@@ -474,7 +474,16 @@ class MediaInfo:
|
||||
self.names = info.get('names') or []
|
||||
# 剩余属性赋值
|
||||
for key, value in info.items():
|
||||
if hasattr(self, key) and not getattr(self, key):
|
||||
if not value:
|
||||
continue
|
||||
if not hasattr(self, key):
|
||||
continue
|
||||
current_value = getattr(self, key)
|
||||
if current_value:
|
||||
continue
|
||||
if current_value is None:
|
||||
setattr(self, key, value)
|
||||
elif type(current_value) == type(value):
|
||||
setattr(self, key, value)
|
||||
|
||||
def set_douban_info(self, info: dict):
|
||||
@@ -606,7 +615,16 @@ class MediaInfo:
|
||||
self.production_countries = [{"id": country, "name": country} for country in info.get("countries") or []]
|
||||
# 剩余属性赋值
|
||||
for key, value in info.items():
|
||||
if not value:
|
||||
continue
|
||||
if not hasattr(self, key):
|
||||
continue
|
||||
current_value = getattr(self, key)
|
||||
if current_value:
|
||||
continue
|
||||
if current_value is None:
|
||||
setattr(self, key, value)
|
||||
elif type(current_value) == type(value):
|
||||
setattr(self, key, value)
|
||||
|
||||
def set_bangumi_info(self, info: dict):
|
||||
|
||||
@@ -69,9 +69,6 @@ class EventManager(metaclass=Singleton):
|
||||
EventManager 负责管理和调度广播事件和链式事件,包括订阅、发送和处理事件
|
||||
"""
|
||||
|
||||
# 退出事件
|
||||
__event = threading.Event()
|
||||
|
||||
def __init__(self):
|
||||
self.__executor = ThreadHelper() # 动态线程池,用于消费事件
|
||||
self.__consumer_threads = [] # 用于保存启动的事件消费者线程
|
||||
@@ -81,6 +78,7 @@ class EventManager(metaclass=Singleton):
|
||||
self.__disabled_handlers = set() # 禁用的事件处理器集合
|
||||
self.__disabled_classes = set() # 禁用的事件处理器类集合
|
||||
self.__lock = threading.Lock() # 线程锁
|
||||
self.__event = threading.Event() # 退出事件
|
||||
|
||||
def start(self):
|
||||
"""
|
||||
|
||||
@@ -9,8 +9,6 @@ class CustomizationMatcher(metaclass=Singleton):
|
||||
"""
|
||||
识别自定义占位符
|
||||
"""
|
||||
customization = None
|
||||
custom_separator = None
|
||||
|
||||
def __init__(self):
|
||||
self.systemconfig = SystemConfigOper()
|
||||
|
||||
@@ -9,7 +9,6 @@ class ReleaseGroupsMatcher(metaclass=Singleton):
|
||||
"""
|
||||
识别制作组、字幕组
|
||||
"""
|
||||
__release_groups: str = None
|
||||
# 内置组
|
||||
RELEASE_GROUPS: dict = {
|
||||
"0ff": ['FF(?:(?:A|WE)B|CD|E(?:DU|B)|TV)'],
|
||||
@@ -48,7 +47,7 @@ class ReleaseGroupsMatcher(metaclass=Singleton):
|
||||
"joyhd": [],
|
||||
"keepfrds": ['FRDS', 'Yumi', 'cXcY'],
|
||||
"lemonhd": ['L(?:eague(?:(?:C|H)D|(?:M|T)V|NF|WEB)|HD)', 'i18n', 'CiNT'],
|
||||
"mteam": ['MTeam(?:TV|)', 'MPAD'],
|
||||
"mteam": ['MTeam(?:TV|)', 'MPAD', 'MWeb'],
|
||||
"nanyangpt": [],
|
||||
"nicept": [],
|
||||
"oshen": [],
|
||||
@@ -70,7 +69,7 @@ class ReleaseGroupsMatcher(metaclass=Singleton):
|
||||
"U2": [],
|
||||
"ultrahd": [],
|
||||
"others": ['B(?:MDru|eyondHD|TN)', 'C(?:fandora|trlhd|MRG)', 'DON', 'EVO', 'FLUX', 'HONE(?:yG|)',
|
||||
'N(?:oGroup|T(?:b|G))', 'PandaMoon', 'SMURF', 'T(?:EPES|aengoo|rollHD )',],
|
||||
'N(?:oGroup|T(?:b|G))', 'PandaMoon', 'SMURF', 'T(?:EPES|aengoo|rollHD )'],
|
||||
"anime": ['ANi', 'HYSUB', 'KTXP', 'LoliHouse', 'MCE', 'Nekomoe kissaten', 'SweetSub', 'MingY',
|
||||
'(?:Lilith|NC)-Raws', '织梦字幕组', '枫叶字幕组', '猎户手抄部', '喵萌奶茶屋', '漫猫字幕社',
|
||||
'霜庭云花Sub', '北宇治字幕组', '氢气烤肉架', '云歌字幕组', '萌樱字幕组', '极影字幕社',
|
||||
|
||||
@@ -312,4 +312,3 @@ class StreamingPlatforms(metaclass=Singleton):
|
||||
if name is None:
|
||||
return False
|
||||
return name.upper() in self._lookup_cache
|
||||
|
||||
|
||||
@@ -16,14 +16,14 @@ class ModuleManager(metaclass=Singleton):
|
||||
模块管理器
|
||||
"""
|
||||
|
||||
# 模块列表
|
||||
_modules: dict = {}
|
||||
# 运行态模块列表
|
||||
_running_modules: dict = {}
|
||||
# 子模块类型集合
|
||||
SubType = Union[DownloaderType, MediaServerType, MessageChannel, StorageSchema, OtherModulesType]
|
||||
|
||||
def __init__(self):
|
||||
# 模块列表
|
||||
self._modules: dict = {}
|
||||
# 运行态模块列表
|
||||
self._running_modules: dict = {}
|
||||
self.load_modules()
|
||||
|
||||
def load_modules(self):
|
||||
|
||||
@@ -10,6 +10,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Type, Union, Callable, Tuple
|
||||
|
||||
from app.helper.sites import SitesHelper
|
||||
from fastapi import HTTPException
|
||||
from starlette import status
|
||||
from watchdog.events import FileSystemEventHandler
|
||||
@@ -21,7 +22,6 @@ from app.core.event import eventmanager, Event
|
||||
from app.db.plugindata_oper import PluginDataOper
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.helper.plugin import PluginHelper
|
||||
from app.helper.sites import SitesHelper
|
||||
from app.log import logger
|
||||
from app.schemas.types import EventType, SystemConfigKey
|
||||
from app.utils.crypto import RSAUtils
|
||||
@@ -88,16 +88,15 @@ class PluginManager(metaclass=Singleton):
|
||||
插件管理器
|
||||
"""
|
||||
|
||||
# 插件列表
|
||||
_plugins: dict = {}
|
||||
# 运行态插件列表
|
||||
_running_plugins: dict = {}
|
||||
# 配置Key
|
||||
_config_key: str = "plugin.%s"
|
||||
# 监听器
|
||||
_observer: Observer = None
|
||||
|
||||
def __init__(self):
|
||||
# 插件列表
|
||||
self._plugins: dict = {}
|
||||
# 运行态插件列表
|
||||
self._running_plugins: dict = {}
|
||||
# 配置Key
|
||||
self._config_key: str = "plugin.%s"
|
||||
# 监听器
|
||||
self._observer: Observer = None
|
||||
# 开发者模式监测插件修改
|
||||
if settings.DEV or settings.PLUGIN_AUTO_RELOAD:
|
||||
self.__start_monitor()
|
||||
@@ -1033,6 +1032,8 @@ class PluginManager(metaclass=Singleton):
|
||||
ret_plugins = []
|
||||
add_time = len(online_plugins)
|
||||
for pid, plugin_info in online_plugins.items():
|
||||
if not isinstance(plugin_info, dict):
|
||||
continue
|
||||
# 如 package_version 为空,则需要判断插件是否兼容当前版本
|
||||
if not package_version:
|
||||
if plugin_info.get(settings.VERSION_FLAG) is not True:
|
||||
|
||||
@@ -1,10 +1,16 @@
|
||||
import threading
|
||||
from time import sleep
|
||||
from typing import Dict, Any, Tuple, List
|
||||
from typing import Dict, Any, Optional
|
||||
from typing import List, Tuple
|
||||
|
||||
from app.core.config import global_vars
|
||||
from app.core.event import eventmanager, Event
|
||||
from app.db.models import Workflow
|
||||
from app.db.workflow_oper import WorkflowOper
|
||||
from app.helper.module import ModuleHelper
|
||||
from app.log import logger
|
||||
from app.schemas import Action, ActionContext
|
||||
from app.schemas import ActionContext, Action
|
||||
from app.schemas.types import EventType
|
||||
from app.utils.singleton import Singleton
|
||||
|
||||
|
||||
@@ -13,10 +19,11 @@ class WorkFlowManager(metaclass=Singleton):
|
||||
工作流管理器
|
||||
"""
|
||||
|
||||
# 所有动作定义
|
||||
_actions: Dict[str, Any] = {}
|
||||
|
||||
def __init__(self):
|
||||
# 所有动作定义
|
||||
self._lock = threading.Lock()
|
||||
self._actions: Dict[str, Any] = {}
|
||||
self._event_workflows: Dict[str, List[int]] = {}
|
||||
self.init()
|
||||
|
||||
def init(self):
|
||||
@@ -49,11 +56,15 @@ class WorkFlowManager(metaclass=Singleton):
|
||||
except Exception as err:
|
||||
logger.error(f"加载动作失败: {action.__name__} - {err}")
|
||||
|
||||
# 加载工作流事件触发器
|
||||
self.load_workflow_events()
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
停止
|
||||
"""
|
||||
pass
|
||||
self._actions = {}
|
||||
self._event_workflows = {}
|
||||
|
||||
def excute(self, workflow_id: int, action: Action,
|
||||
context: ActionContext = None) -> Tuple[bool, str, ActionContext]:
|
||||
@@ -110,3 +121,180 @@ class WorkFlowManager(metaclass=Singleton):
|
||||
}
|
||||
} for key, action in self._actions.items()
|
||||
]
|
||||
|
||||
def update_workflow_event(self, workflow: Workflow):
|
||||
"""
|
||||
更新工作流事件触发器
|
||||
"""
|
||||
# 确保先移除旧的事件监听器
|
||||
self.remove_workflow_event(workflow_id=workflow.id, event_type_str=workflow.event_type)
|
||||
# 如果工作流是事件触发类型且未被禁用
|
||||
if workflow.trigger_type == "event" and workflow.state != 'P':
|
||||
# 注册事件触发器
|
||||
self.register_workflow_event(workflow.id, workflow.event_type)
|
||||
|
||||
def load_workflow_events(self, workflow_id: Optional[int] = None):
|
||||
"""
|
||||
加载工作流触发事件
|
||||
"""
|
||||
workflows = []
|
||||
if workflow_id:
|
||||
workflow = WorkflowOper().get(workflow_id)
|
||||
if workflow:
|
||||
workflows = [workflow]
|
||||
else:
|
||||
workflows = WorkflowOper().get_event_triggered_workflows()
|
||||
try:
|
||||
for workflow in workflows:
|
||||
self.update_workflow_event(workflow)
|
||||
except Exception as e:
|
||||
logger.error(f"加载事件触发工作流失败: {e}")
|
||||
|
||||
def register_workflow_event(self, workflow_id: int, event_type_str: str):
|
||||
"""
|
||||
注册工作流事件触发器
|
||||
"""
|
||||
try:
|
||||
event_type = EventType(event_type_str)
|
||||
except ValueError:
|
||||
logger.error(f"无效的事件类型: {event_type_str}")
|
||||
return
|
||||
if event_type in EventType:
|
||||
# 确保先移除旧的事件监听器
|
||||
self.remove_workflow_event(workflow_id, event_type.value)
|
||||
with self._lock:
|
||||
# 添加新的事件监听器
|
||||
eventmanager.add_event_listener(event_type, self._handle_event)
|
||||
# 记录工作流事件触发器
|
||||
if event_type.value not in self._event_workflows:
|
||||
self._event_workflows[event_type.value] = []
|
||||
self._event_workflows[event_type.value].append(workflow_id)
|
||||
logger.info(f"已注册工作流 {workflow_id} 事件触发器: {event_type.value}")
|
||||
|
||||
def remove_workflow_event(self, workflow_id: int, event_type_str: str):
|
||||
"""
|
||||
移除工作流事件触发器
|
||||
"""
|
||||
try:
|
||||
event_type = EventType(event_type_str)
|
||||
except ValueError:
|
||||
logger.error(f"无效的事件类型: {event_type_str}")
|
||||
return
|
||||
if event_type in EventType:
|
||||
with self._lock:
|
||||
eventmanager.remove_event_listener(event_type, self._handle_event)
|
||||
if event_type.value in self._event_workflows:
|
||||
if workflow_id in self._event_workflows[event_type.value]:
|
||||
self._event_workflows[event_type.value].remove(workflow_id)
|
||||
if not self._event_workflows[event_type.value]:
|
||||
del self._event_workflows[event_type.value]
|
||||
logger.info(f"已移除工作流 {workflow_id} 事件触发器")
|
||||
|
||||
def _handle_event(self, event: Event):
|
||||
"""
|
||||
处理事件,触发相应的工作流
|
||||
"""
|
||||
try:
|
||||
event_type_str = str(event.event_type.value)
|
||||
with self._lock:
|
||||
if event_type_str not in self._event_workflows:
|
||||
return
|
||||
workflow_ids = self._event_workflows[event_type_str].copy()
|
||||
for workflow_id in workflow_ids:
|
||||
self._trigger_workflow(workflow_id, event)
|
||||
except Exception as e:
|
||||
logger.error(f"处理工作流事件失败: {e}")
|
||||
|
||||
def _trigger_workflow(self, workflow_id: int, event: Event):
|
||||
"""
|
||||
触发工作流执行
|
||||
"""
|
||||
try:
|
||||
# 检查工作流是否存在且启用
|
||||
workflow = WorkflowOper().get(workflow_id)
|
||||
if not workflow or workflow.state == 'P':
|
||||
return
|
||||
|
||||
# 检查事件条件
|
||||
if not self._check_event_conditions(workflow, event):
|
||||
logger.debug(f"工作流 {workflow.name} 事件条件不匹配,跳过执行")
|
||||
return
|
||||
|
||||
# 检查工作流是否正在运行
|
||||
if workflow.state == 'R':
|
||||
logger.warning(f"工作流 {workflow.name} 正在运行中,跳过重复触发")
|
||||
return
|
||||
|
||||
logger.info(f"事件 {event.event_type.value} 触发工作流: {workflow.name}")
|
||||
|
||||
# 发送工作流执行事件以启动工作流
|
||||
eventmanager.send_event(EventType.WorkflowExecute, {
|
||||
"workflow_id": workflow_id,
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"触发工作流 {workflow_id} 失败: {e}")
|
||||
|
||||
def _check_event_conditions(self, workflow, event: Event) -> bool:
|
||||
"""
|
||||
检查事件是否满足工作流的触发条件
|
||||
"""
|
||||
if not workflow.event_conditions:
|
||||
return True
|
||||
|
||||
conditions = workflow.event_conditions
|
||||
event_data = event.event_data or {}
|
||||
|
||||
# 检查字段匹配条件
|
||||
for field, expected_value in conditions.items():
|
||||
if field not in event_data:
|
||||
return False
|
||||
actual_value = event_data[field]
|
||||
# 支持多种条件匹配方式
|
||||
if isinstance(expected_value, dict):
|
||||
# 复杂条件匹配
|
||||
if not self._check_complex_condition(actual_value, expected_value):
|
||||
return False
|
||||
else:
|
||||
# 简单值匹配
|
||||
if actual_value != expected_value:
|
||||
return False
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _check_complex_condition(actual_value: any, condition: dict) -> bool:
|
||||
"""
|
||||
检查复杂条件匹配
|
||||
支持的操作符:equals, not_equals, contains, not_contains, in, not_in, regex
|
||||
"""
|
||||
for operator, expected_value in condition.items():
|
||||
if operator == "equals":
|
||||
if actual_value != expected_value:
|
||||
return False
|
||||
elif operator == "not_equals":
|
||||
if actual_value == expected_value:
|
||||
return False
|
||||
elif operator == "contains":
|
||||
if expected_value not in str(actual_value):
|
||||
return False
|
||||
elif operator == "not_contains":
|
||||
if expected_value in str(actual_value):
|
||||
return False
|
||||
elif operator == "in":
|
||||
if actual_value not in expected_value:
|
||||
return False
|
||||
elif operator == "not_in":
|
||||
if actual_value in expected_value:
|
||||
return False
|
||||
elif operator == "regex":
|
||||
import re
|
||||
if not re.search(expected_value, str(actual_value)):
|
||||
return False
|
||||
return True
|
||||
|
||||
def get_event_workflows(self) -> dict:
|
||||
"""
|
||||
获取所有事件触发的工作流
|
||||
"""
|
||||
with self._lock:
|
||||
return self._event_workflows.copy()
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import Column, Integer, JSON, Sequence, String, and_
|
||||
from sqlalchemy import Column, Integer, JSON, Sequence, String, and_, or_
|
||||
|
||||
from app.db import Base, db_query, db_update
|
||||
|
||||
@@ -18,6 +18,12 @@ class Workflow(Base):
|
||||
description = Column(String)
|
||||
# 定时器
|
||||
timer = Column(String)
|
||||
# 触发类型:timer-定时触发 event-事件触发 manual-手动触发
|
||||
trigger_type = Column(String, default='timer')
|
||||
# 事件类型(当trigger_type为event时使用)
|
||||
event_type = Column(String)
|
||||
# 事件条件(JSON格式,用于过滤事件)
|
||||
event_conditions = Column(JSON, default=dict)
|
||||
# 状态:W-等待 R-运行中 P-暂停 S-成功 F-失败
|
||||
state = Column(String, nullable=False, index=True, default='W')
|
||||
# 已执行动作(,分隔)
|
||||
@@ -37,11 +43,41 @@ class Workflow(Base):
|
||||
# 最后执行时间
|
||||
last_time = Column(String)
|
||||
|
||||
@staticmethod
|
||||
@db_query
|
||||
def list(db):
|
||||
return db.query(Workflow).all()
|
||||
|
||||
@staticmethod
|
||||
@db_query
|
||||
def get_enabled_workflows(db):
|
||||
return db.query(Workflow).filter(Workflow.state != 'P').all()
|
||||
|
||||
@staticmethod
|
||||
@db_query
|
||||
def get_timer_triggered_workflows(db):
|
||||
"""获取定时触发的工作流"""
|
||||
return db.query(Workflow).filter(
|
||||
and_(
|
||||
or_(
|
||||
Workflow.trigger_type == 'timer',
|
||||
not Workflow.trigger_type
|
||||
),
|
||||
Workflow.state != 'P'
|
||||
)
|
||||
).all()
|
||||
|
||||
@staticmethod
|
||||
@db_query
|
||||
def get_event_triggered_workflows(db):
|
||||
"""获取事件触发的工作流"""
|
||||
return db.query(Workflow).filter(
|
||||
and_(
|
||||
Workflow.trigger_type == 'event',
|
||||
Workflow.state != 'P'
|
||||
)
|
||||
).all()
|
||||
|
||||
@staticmethod
|
||||
@db_query
|
||||
def get_by_name(db, name: str):
|
||||
|
||||
@@ -25,12 +25,30 @@ class WorkflowOper(DbOper):
|
||||
"""
|
||||
return Workflow.get(self._db, wid)
|
||||
|
||||
def list(self) -> List[Workflow]:
|
||||
"""
|
||||
获取所有工作流列表
|
||||
"""
|
||||
return Workflow.list(self._db)
|
||||
|
||||
def list_enabled(self) -> List[Workflow]:
|
||||
"""
|
||||
获取启用的工作流列表
|
||||
"""
|
||||
return Workflow.get_enabled_workflows(self._db)
|
||||
|
||||
def get_timer_triggered_workflows(self) -> List[Workflow]:
|
||||
"""
|
||||
获取定时触发的工作流列表
|
||||
"""
|
||||
return Workflow.get_timer_triggered_workflows(self._db)
|
||||
|
||||
def get_event_triggered_workflows(self) -> List[Workflow]:
|
||||
"""
|
||||
获取事件触发的工作流列表
|
||||
"""
|
||||
return Workflow.get_event_triggered_workflows(self._db)
|
||||
|
||||
def get_by_name(self, name: str) -> Workflow:
|
||||
"""
|
||||
按名称获取工作流
|
||||
|
||||
@@ -96,53 +96,58 @@ class CookieHelper:
|
||||
return None, None, "获取源码失败"
|
||||
# 查找用户名输入框
|
||||
html = etree.HTML(html_text)
|
||||
username_xpath = None
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("username"):
|
||||
if html.xpath(xpath):
|
||||
username_xpath = xpath
|
||||
break
|
||||
if not username_xpath:
|
||||
return None, None, "未找到用户名输入框"
|
||||
# 查找密码输入框
|
||||
password_xpath = None
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("password"):
|
||||
if html.xpath(xpath):
|
||||
password_xpath = xpath
|
||||
break
|
||||
if not password_xpath:
|
||||
return None, None, "未找到密码输入框"
|
||||
# 处理二步验证码
|
||||
otp_code = TwoFactorAuth(two_step_code).get_code()
|
||||
# 查找二步验证码输入框
|
||||
twostep_xpath = None
|
||||
if otp_code:
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("twostep"):
|
||||
try:
|
||||
username_xpath = None
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("username"):
|
||||
if html.xpath(xpath):
|
||||
twostep_xpath = xpath
|
||||
username_xpath = xpath
|
||||
break
|
||||
# 查找验证码输入框
|
||||
captcha_xpath = None
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("captcha"):
|
||||
if html.xpath(xpath):
|
||||
captcha_xpath = xpath
|
||||
break
|
||||
# 查找验证码图片
|
||||
captcha_img_url = None
|
||||
if captcha_xpath:
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("captcha_img"):
|
||||
if not username_xpath:
|
||||
return None, None, "未找到用户名输入框"
|
||||
# 查找密码输入框
|
||||
password_xpath = None
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("password"):
|
||||
if html.xpath(xpath):
|
||||
captcha_img_url = html.xpath(xpath)[0]
|
||||
password_xpath = xpath
|
||||
break
|
||||
if not captcha_img_url:
|
||||
return None, None, "未找到验证码图片"
|
||||
# 查找登录按钮
|
||||
submit_xpath = None
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("submit"):
|
||||
if html.xpath(xpath):
|
||||
submit_xpath = xpath
|
||||
break
|
||||
if not submit_xpath:
|
||||
return None, None, "未找到登录按钮"
|
||||
if not password_xpath:
|
||||
return None, None, "未找到密码输入框"
|
||||
# 处理二步验证码
|
||||
otp_code = TwoFactorAuth(two_step_code).get_code()
|
||||
# 查找二步验证码输入框
|
||||
twostep_xpath = None
|
||||
if otp_code:
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("twostep"):
|
||||
if html.xpath(xpath):
|
||||
twostep_xpath = xpath
|
||||
break
|
||||
# 查找验证码输入框
|
||||
captcha_xpath = None
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("captcha"):
|
||||
if html.xpath(xpath):
|
||||
captcha_xpath = xpath
|
||||
break
|
||||
# 查找验证码图片
|
||||
captcha_img_url = None
|
||||
if captcha_xpath:
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("captcha_img"):
|
||||
if html.xpath(xpath):
|
||||
captcha_img_url = html.xpath(xpath)[0]
|
||||
break
|
||||
if not captcha_img_url:
|
||||
return None, None, "未找到验证码图片"
|
||||
# 查找登录按钮
|
||||
submit_xpath = None
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("submit"):
|
||||
if html.xpath(xpath):
|
||||
submit_xpath = xpath
|
||||
break
|
||||
if not submit_xpath:
|
||||
return None, None, "未找到登录按钮"
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
# 点击登录按钮
|
||||
try:
|
||||
# 等待登录按钮准备好
|
||||
@@ -185,19 +190,23 @@ class CookieHelper:
|
||||
if not otp_code:
|
||||
return None, None, "需要二次验证码"
|
||||
html = etree.HTML(page.content())
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("twostep"):
|
||||
if html.xpath(xpath):
|
||||
try:
|
||||
# 刷新一下 2fa code
|
||||
otp_code = TwoFactorAuth(two_step_code).get_code()
|
||||
page.fill(xpath, otp_code)
|
||||
# 登录按钮 xpath 理论上相同,不再重复查找
|
||||
page.click(submit_xpath)
|
||||
page.wait_for_load_state("networkidle", timeout=30 * 1000)
|
||||
except Exception as e:
|
||||
logger.error(f"二次验证码输入失败:{str(e)}")
|
||||
return None, None, f"二次验证码输入失败:{str(e)}"
|
||||
break
|
||||
try:
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("twostep"):
|
||||
if html.xpath(xpath):
|
||||
try:
|
||||
# 刷新一下 2fa code
|
||||
otp_code = TwoFactorAuth(two_step_code).get_code()
|
||||
page.fill(xpath, otp_code)
|
||||
# 登录按钮 xpath 理论上相同,不再重复查找
|
||||
page.click(submit_xpath)
|
||||
page.wait_for_load_state("networkidle", timeout=30 * 1000)
|
||||
except Exception as e:
|
||||
logger.error(f"二次验证码输入失败:{str(e)}")
|
||||
return None, None, f"二次验证码输入失败:{str(e)}"
|
||||
break
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
# 登录后的源码
|
||||
html_text = page.content()
|
||||
if not html_text:
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from app import schemas
|
||||
from app.core.context import MediaInfo
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.log import logger
|
||||
from app.schemas.types import SystemConfigKey
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
JINJA2_VAR_PATTERN = re.compile(r"\{\{.*?\}\}", re.DOTALL)
|
||||
|
||||
|
||||
class DirectoryHelper:
|
||||
"""
|
||||
@@ -109,3 +113,42 @@ class DirectoryHelper:
|
||||
return matched_dir
|
||||
return matched_dirs[0]
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_media_root_path(rename_format: str, rename_path: Path) -> Optional[Path]:
|
||||
"""
|
||||
获取重命名后的媒体文件根路径
|
||||
|
||||
:param rename_format: 重命名格式
|
||||
:param rename_path: 重命名后的路径
|
||||
:return: 媒体文件根路径
|
||||
"""
|
||||
if not rename_format:
|
||||
logger.error("重命名格式不能为空")
|
||||
return None
|
||||
# 计算重命名中的文件夹层数
|
||||
rename_list = rename_format.split("/")
|
||||
rename_format_level = len(rename_list) - 1
|
||||
# 查找标题参数所在层
|
||||
for level, name in enumerate(rename_list):
|
||||
matchs = JINJA2_VAR_PATTERN.findall(name)
|
||||
if not matchs:
|
||||
continue
|
||||
# 处理特例,有的人重命名的第一层是年份、分辨率
|
||||
if any("title" in m for m in matchs):
|
||||
# 找出含标题的这一层作为媒体根目录
|
||||
rename_format_level -= level
|
||||
break
|
||||
else:
|
||||
# 假定第一层目录是媒体根目录
|
||||
logger.warn(f"重命名格式 {rename_format} 缺少标题参数")
|
||||
if rename_format_level > len(rename_path.parents):
|
||||
# 通常因为路径以/结尾,被Path规范化删除了
|
||||
logger.error(f"路径 {rename_path} 不匹配重命名格式 {rename_format}")
|
||||
return None
|
||||
if rename_format_level <= 0:
|
||||
# 所有媒体文件都存在一个目录内的特殊需求
|
||||
rename_format_level = 1
|
||||
# 媒体根路径
|
||||
media_root = rename_path.parents[rename_format_level - 1]
|
||||
return media_root
|
||||
|
||||
@@ -8,9 +8,9 @@ import os
|
||||
|
||||
|
||||
class DisplayHelper(metaclass=Singleton):
|
||||
_display: Display = None
|
||||
|
||||
def __init__(self):
|
||||
self._display = None
|
||||
if not SystemUtils.is_docker():
|
||||
return
|
||||
try:
|
||||
|
||||
@@ -70,6 +70,9 @@ def enable_doh(enable: bool):
|
||||
|
||||
|
||||
class DohHelper(metaclass=Singleton):
|
||||
"""
|
||||
DoH帮助类,用于处理DNS over HTTPS解析。
|
||||
"""
|
||||
def __init__(self):
|
||||
enable_doh(settings.DOH_ENABLE)
|
||||
|
||||
|
||||
@@ -1,457 +0,0 @@
|
||||
import gc
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
import psutil
|
||||
from pympler import muppy, summary, asizeof
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.event import eventmanager, Event
|
||||
from app.log import logger
|
||||
from app.schemas import ConfigChangeEventData
|
||||
from app.schemas.types import EventType
|
||||
from app.utils.singleton import Singleton
|
||||
|
||||
|
||||
class MemoryHelper(metaclass=Singleton):
|
||||
"""
|
||||
内存管理工具类,用于监控和优化内存使用
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# 检查间隔(秒) - 从配置获取,默认5分钟
|
||||
self._check_interval = settings.MEMORY_SNAPSHOT_INTERVAL * 60
|
||||
self._monitoring = False
|
||||
self._monitor_thread: Optional[threading.Thread] = None
|
||||
# 内存快照保存目录
|
||||
self._memory_snapshot_dir = settings.LOG_PATH / "memory_snapshots"
|
||||
# 保留的快照文件数量
|
||||
self._keep_count = settings.MEMORY_SNAPSHOT_KEEP_COUNT
|
||||
|
||||
@eventmanager.register(EventType.ConfigChanged)
|
||||
def handle_config_changed(self, event: Event):
|
||||
"""
|
||||
处理配置变更事件,更新内存监控设置
|
||||
:param event: 事件对象
|
||||
"""
|
||||
if not event:
|
||||
return
|
||||
event_data: ConfigChangeEventData = event.event_data
|
||||
if event_data.key not in ['MEMORY_ANALYSIS', 'MEMORY_SNAPSHOT_INTERVAL', 'MEMORY_SNAPSHOT_KEEP_COUNT']:
|
||||
return
|
||||
|
||||
# 更新配置
|
||||
if event_data.key == 'MEMORY_SNAPSHOT_INTERVAL':
|
||||
self._check_interval = settings.MEMORY_SNAPSHOT_INTERVAL * 60
|
||||
elif event_data.key == 'MEMORY_SNAPSHOT_KEEP_COUNT':
|
||||
self._keep_count = settings.MEMORY_SNAPSHOT_KEEP_COUNT
|
||||
self.stop_monitoring()
|
||||
self.start_monitoring()
|
||||
|
||||
def start_monitoring(self):
|
||||
"""
|
||||
开始内存监控
|
||||
"""
|
||||
if not settings.MEMORY_ANALYSIS:
|
||||
return
|
||||
if self._monitoring:
|
||||
return
|
||||
|
||||
# 创建内存快照目录
|
||||
self._memory_snapshot_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 初始化内存分析器
|
||||
self._monitoring = True
|
||||
self._monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True)
|
||||
self._monitor_thread.start()
|
||||
logger.info("内存监控已启动")
|
||||
|
||||
def stop_monitoring(self):
|
||||
"""
|
||||
停止内存监控
|
||||
"""
|
||||
self._monitoring = False
|
||||
if self._monitor_thread:
|
||||
self._monitor_thread.join(timeout=5)
|
||||
logger.info("内存监控已停止")
|
||||
|
||||
def _monitor_loop(self):
|
||||
"""
|
||||
内存监控循环
|
||||
"""
|
||||
logger.info("内存监控循环开始")
|
||||
while self._monitoring:
|
||||
try:
|
||||
# 生成内存快照
|
||||
self._create_memory_snapshot()
|
||||
time.sleep(self._check_interval)
|
||||
except Exception as e:
|
||||
logger.error(f"内存监控出错: {e}")
|
||||
# 出错后等待1分钟再继续
|
||||
time.sleep(60)
|
||||
logger.info("内存监控循环结束")
|
||||
|
||||
def _create_memory_snapshot(self):
|
||||
"""
|
||||
创建内存快照并保存到文件
|
||||
"""
|
||||
try:
|
||||
# 获取当前时间戳
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
snapshot_file = self._memory_snapshot_dir / f"memory_snapshot_{timestamp}.txt"
|
||||
|
||||
# 获取系统内存使用情况
|
||||
memory_usage = psutil.Process().memory_info().rss
|
||||
|
||||
logger.info(f"开始创建内存快照: {snapshot_file}")
|
||||
|
||||
# 第一步:写入基本信息和对象类型统计
|
||||
self._write_basic_info(snapshot_file, memory_usage)
|
||||
|
||||
# 第二步:分析并写入类实例内存使用情况
|
||||
self._append_class_analysis(snapshot_file)
|
||||
|
||||
# 第三步:分析并写入大内存变量详情
|
||||
self._append_variable_analysis(snapshot_file)
|
||||
|
||||
logger.info(f"内存快照已保存: {snapshot_file}, 当前内存使用: {memory_usage / 1024 / 1024:.2f} MB")
|
||||
|
||||
# 清理过期的快照文件(保留最近30个)
|
||||
self._cleanup_old_snapshots()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"创建内存快照失败: {e}")
|
||||
|
||||
@staticmethod
|
||||
def _write_basic_info(snapshot_file, memory_usage):
|
||||
"""
|
||||
写入基本信息和对象类型统计
|
||||
"""
|
||||
# 获取当前进程的内存使用情况
|
||||
all_objects = muppy.get_objects()
|
||||
sum1 = summary.summarize(all_objects)
|
||||
|
||||
with open(snapshot_file, 'w', encoding='utf-8') as f:
|
||||
f.write(f"内存快照时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
|
||||
f.write(f"当前进程内存使用: {memory_usage / 1024 / 1024:.2f} MB\n")
|
||||
f.write("=" * 80 + "\n")
|
||||
f.write("对象类型统计:\n")
|
||||
f.write("-" * 80 + "\n")
|
||||
|
||||
# 写入对象统计信息
|
||||
for line in summary.format_(sum1):
|
||||
f.write(line + "\n")
|
||||
|
||||
# 立即刷新到磁盘
|
||||
f.flush()
|
||||
|
||||
logger.debug("基本信息已写入快照文件")
|
||||
|
||||
def _append_class_analysis(self, snapshot_file):
|
||||
"""
|
||||
分析并追加类实例内存使用情况
|
||||
"""
|
||||
with open(snapshot_file, 'a', encoding='utf-8') as f:
|
||||
f.write("\n" + "=" * 80 + "\n")
|
||||
f.write("类实例内存使用情况 (按内存大小排序):\n")
|
||||
f.write("-" * 80 + "\n")
|
||||
f.write("正在分析中...\n")
|
||||
# 立即刷新,让用户知道这部分开始了
|
||||
f.flush()
|
||||
|
||||
try:
|
||||
logger.debug("开始分析类实例内存使用情况")
|
||||
class_objects = self._get_class_memory_usage()
|
||||
|
||||
# 重新打开文件,移除"正在分析中..."并写入实际结果
|
||||
with open(snapshot_file, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# 替换"正在分析中..."
|
||||
content = content.replace("正在分析中...\n", "")
|
||||
|
||||
with open(snapshot_file, 'w', encoding='utf-8') as f:
|
||||
f.write(content)
|
||||
|
||||
if class_objects:
|
||||
# 只显示前100个类
|
||||
for i, class_info in enumerate(class_objects[:100], 1):
|
||||
f.write(f"{i:3d}. {class_info['name']:<50} "
|
||||
f"{class_info['size_mb']:>8.2f} MB ({class_info['count']} 个实例)\n")
|
||||
else:
|
||||
f.write("未找到有效的类实例信息\n")
|
||||
|
||||
f.flush()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"获取类实例信息失败: {e}")
|
||||
|
||||
# 即使出错也要更新文件
|
||||
with open(snapshot_file, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
content = content.replace("正在分析中...\n", f"获取类实例信息失败: {e}\n")
|
||||
|
||||
with open(snapshot_file, 'w', encoding='utf-8') as f:
|
||||
f.write(content)
|
||||
f.flush()
|
||||
|
||||
logger.debug("类实例分析已完成并写入")
|
||||
|
||||
def _append_variable_analysis(self, snapshot_file):
|
||||
"""
|
||||
分析并追加大内存变量详情
|
||||
"""
|
||||
with open(snapshot_file, 'a', encoding='utf-8') as f:
|
||||
f.write("\n" + "=" * 80 + "\n")
|
||||
f.write("大内存变量详情 (前100个):\n")
|
||||
f.write("-" * 80 + "\n")
|
||||
f.write("正在分析中...\n")
|
||||
# 立即刷新,让用户知道这部分开始了
|
||||
f.flush()
|
||||
|
||||
try:
|
||||
logger.debug("开始分析大内存变量")
|
||||
large_variables = self._get_large_variables(100)
|
||||
|
||||
# 重新打开文件,移除"正在分析中..."并写入实际结果
|
||||
with open(snapshot_file, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# 替换最后的"正在分析中..."
|
||||
content = content.replace("正在分析中...\n", "")
|
||||
|
||||
with open(snapshot_file, 'w', encoding='utf-8') as f:
|
||||
f.write(content)
|
||||
|
||||
if large_variables:
|
||||
for i, var_info in enumerate(large_variables, 1):
|
||||
f.write(
|
||||
f"{i:3d}. {var_info['name']:<30} {var_info['type']:<15} {var_info['size_mb']:>8.2f} MB\n")
|
||||
else:
|
||||
f.write("未找到大内存变量\n")
|
||||
|
||||
f.flush()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"获取大内存变量信息失败: {e}")
|
||||
|
||||
# 即使出错也要更新文件
|
||||
with open(snapshot_file, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
content = content.replace("正在分析中...\n", f"获取变量信息失败: {e}\n")
|
||||
|
||||
with open(snapshot_file, 'w', encoding='utf-8') as f:
|
||||
f.write(content)
|
||||
f.flush()
|
||||
|
||||
logger.debug("大内存变量分析已完成并写入")
|
||||
|
||||
def _cleanup_old_snapshots(self):
|
||||
"""
|
||||
清理过期的内存快照文件,只保留最近的指定数量文件
|
||||
"""
|
||||
try:
|
||||
snapshot_files = list(self._memory_snapshot_dir.glob("memory_snapshot_*.txt"))
|
||||
if len(snapshot_files) > self._keep_count:
|
||||
# 按修改时间排序,删除最旧的文件
|
||||
snapshot_files.sort(key=lambda x: x.stat().st_mtime)
|
||||
for old_file in snapshot_files[:-self._keep_count]:
|
||||
old_file.unlink()
|
||||
logger.debug(f"已删除过期内存快照: {old_file}")
|
||||
except Exception as e:
|
||||
logger.error(f"清理过期快照失败: {e}")
|
||||
|
||||
@staticmethod
|
||||
def _get_class_memory_usage():
|
||||
"""
|
||||
获取所有类实例的内存使用情况,按内存大小排序
|
||||
"""
|
||||
class_info = {}
|
||||
processed_count = 0
|
||||
error_count = 0
|
||||
|
||||
# 获取所有对象
|
||||
all_objects = muppy.get_objects()
|
||||
logger.debug(f"开始分析 {len(all_objects)} 个对象的类实例内存使用情况")
|
||||
|
||||
for obj in all_objects:
|
||||
try:
|
||||
# 跳过类对象本身,统计类的实例
|
||||
if isinstance(obj, type):
|
||||
continue
|
||||
|
||||
# 获取对象的类名 - 这里可能会出错
|
||||
obj_class = type(obj)
|
||||
|
||||
# 安全地获取类名
|
||||
try:
|
||||
if hasattr(obj_class, '__module__') and hasattr(obj_class, '__name__'):
|
||||
class_name = f"{obj_class.__module__}.{obj_class.__name__}"
|
||||
else:
|
||||
class_name = str(obj_class)
|
||||
except Exception as e:
|
||||
# 如果获取类名失败,使用简单的类型描述
|
||||
class_name = f"<unknown_class_{id(obj_class)}>"
|
||||
logger.debug(f"获取类名失败: {e}")
|
||||
|
||||
# 计算对象本身的内存使用(不包括引用对象,避免重复计算)
|
||||
size_bytes = sys.getsizeof(obj)
|
||||
if size_bytes < 100: # 跳过太小的对象
|
||||
continue
|
||||
|
||||
size_mb = size_bytes / 1024 / 1024
|
||||
processed_count += 1
|
||||
|
||||
if class_name in class_info:
|
||||
class_info[class_name]['size_mb'] += size_mb
|
||||
class_info[class_name]['count'] += 1
|
||||
else:
|
||||
class_info[class_name] = {
|
||||
'name': class_name,
|
||||
'size_mb': size_mb,
|
||||
'count': 1
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
# 捕获所有可能的异常,包括SQLAlchemy、ORM等框架的异常
|
||||
error_count += 1
|
||||
if error_count <= 5: # 只记录前5个错误,避免日志过多
|
||||
logger.debug(f"分析对象时出错: {e}")
|
||||
continue
|
||||
|
||||
logger.debug(f"类实例分析完成: 处理了 {processed_count} 个对象, 遇到 {error_count} 个错误")
|
||||
|
||||
# 按内存大小排序
|
||||
sorted_classes = sorted(class_info.values(), key=lambda x: x['size_mb'], reverse=True)
|
||||
return sorted_classes
|
||||
|
||||
def _get_large_variables(self, limit=100):
|
||||
"""
|
||||
获取大内存变量信息,按内存大小排序
|
||||
使用已计算对象集合避免重复计算
|
||||
"""
|
||||
large_vars = []
|
||||
processed_count = 0
|
||||
calculated_objects = set() # 避免重复计算
|
||||
|
||||
# 获取所有对象
|
||||
all_objects = muppy.get_objects()
|
||||
logger.debug(f"开始分析 {len(all_objects)} 个对象的内存使用情况")
|
||||
|
||||
for obj in all_objects:
|
||||
# 跳过类对象
|
||||
if isinstance(obj, type):
|
||||
continue
|
||||
|
||||
# 跳过已经计算过的对象
|
||||
obj_id = id(obj)
|
||||
if obj_id in calculated_objects:
|
||||
continue
|
||||
|
||||
try:
|
||||
# 首先使用 sys.getsizeof 快速筛选
|
||||
shallow_size = sys.getsizeof(obj)
|
||||
if shallow_size < 1024: # 只处理大于1KB的对象
|
||||
continue
|
||||
|
||||
# 对于较大的对象,使用 asizeof 进行深度计算
|
||||
size_bytes = asizeof.asizeof(obj)
|
||||
|
||||
# 只处理大于10KB的对象,提高分析效率
|
||||
if size_bytes < 10240:
|
||||
continue
|
||||
|
||||
size_mb = size_bytes / 1024 / 1024
|
||||
processed_count += 1
|
||||
calculated_objects.add(obj_id)
|
||||
|
||||
# 获取对象信息
|
||||
var_info = self._get_variable_info(obj, size_mb)
|
||||
if var_info:
|
||||
large_vars.append(var_info)
|
||||
|
||||
# 如果已经找到足够多的大对象,可以提前结束
|
||||
if len(large_vars) >= limit * 2: # 多收集一些,后面排序筛选
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
# 更广泛的异常捕获
|
||||
logger.debug(f"分析对象失败: {e}")
|
||||
continue
|
||||
|
||||
logger.debug(f"处理了 {processed_count} 个大对象,找到 {len(large_vars)} 个有效变量")
|
||||
|
||||
# 按内存大小排序并返回前N个
|
||||
large_vars.sort(key=lambda x: x['size_mb'], reverse=True)
|
||||
return large_vars[:limit]
|
||||
|
||||
def _get_variable_info(self, obj, size_mb):
|
||||
"""
|
||||
获取变量的描述信息
|
||||
"""
|
||||
try:
|
||||
obj_type = type(obj).__name__
|
||||
|
||||
# 尝试获取变量名
|
||||
var_name = self._get_variable_name(obj)
|
||||
|
||||
# 生成描述性信息
|
||||
if isinstance(obj, dict):
|
||||
key_count = len(obj)
|
||||
if key_count > 0:
|
||||
sample_keys = list(obj.keys())[:3]
|
||||
var_name += f" ({key_count}项, 键: {sample_keys})"
|
||||
elif isinstance(obj, (list, tuple, set)):
|
||||
var_name += f" ({len(obj)}个元素)"
|
||||
elif isinstance(obj, str):
|
||||
if len(obj) > 50:
|
||||
var_name += f" (长度: {len(obj)}, 内容: '{obj[:50]}...')"
|
||||
else:
|
||||
var_name += f" ('{obj}')"
|
||||
elif hasattr(obj, '__class__') and hasattr(obj.__class__, '__name__'):
|
||||
if hasattr(obj, '__dict__'):
|
||||
attr_count = len(obj.__dict__)
|
||||
var_name += f" ({attr_count}个属性)"
|
||||
|
||||
return {
|
||||
'name': var_name,
|
||||
'type': obj_type,
|
||||
'size_mb': size_mb
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"获取变量信息失败: {e}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_variable_name(obj):
|
||||
"""
|
||||
尝试获取变量名
|
||||
"""
|
||||
try:
|
||||
# 尝试通过gc获取引用该对象的变量名
|
||||
referrers = gc.get_referrers(obj)
|
||||
|
||||
for referrer in referrers:
|
||||
if isinstance(referrer, dict):
|
||||
# 检查是否在某个模块的全局变量中
|
||||
for name, value in referrer.items():
|
||||
if value is obj and isinstance(name, str):
|
||||
return name
|
||||
elif hasattr(referrer, '__dict__'):
|
||||
# 检查是否在某个实例的属性中
|
||||
for name, value in referrer.__dict__.items():
|
||||
if value is obj and isinstance(name, str):
|
||||
return f"{type(referrer).__name__}.{name}"
|
||||
|
||||
# 如果找不到变量名,返回对象类型和id
|
||||
return f"{type(obj).__name__}_{id(obj)}"
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"获取变量名失败: {e}")
|
||||
return f"{type(obj).__name__}_{id(obj)}"
|
||||
@@ -541,8 +541,6 @@ class MessageQueueManager(metaclass=SingletonClass):
|
||||
消息发送队列管理器
|
||||
"""
|
||||
|
||||
schedule_periods: List[tuple[int, int, int, int]] = []
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
send_callback: Optional[Callable] = None,
|
||||
@@ -554,6 +552,8 @@ class MessageQueueManager(metaclass=SingletonClass):
|
||||
:param send_callback: 实际发送消息的回调函数
|
||||
:param check_interval: 时间检查间隔(秒)
|
||||
"""
|
||||
self.schedule_periods: List[tuple[int, int, int, int]] = []
|
||||
|
||||
self.init_config()
|
||||
|
||||
self.queue: queue.Queue[Any] = queue.Queue()
|
||||
|
||||
@@ -18,14 +18,14 @@ from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.log import logger
|
||||
from app.schemas.types import SystemConfigKey
|
||||
from app.utils.http import RequestUtils
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.singleton import WeakSingleton
|
||||
from app.utils.system import SystemUtils
|
||||
from app.utils.url import UrlUtils
|
||||
|
||||
PLUGIN_DIR = Path(settings.ROOT_PATH) / "app" / "plugins"
|
||||
|
||||
|
||||
class PluginHelper(metaclass=Singleton):
|
||||
class PluginHelper(metaclass=WeakSingleton):
|
||||
"""
|
||||
插件市场管理,下载安装插件到本地
|
||||
"""
|
||||
@@ -649,10 +649,20 @@ class PluginHelper(metaclass=Singleton):
|
||||
"""
|
||||
dependencies = {}
|
||||
try:
|
||||
install_plugins = {
|
||||
plugin_id.lower() # 对应插件的小写目录名
|
||||
for plugin_id in SystemConfigOper().get(
|
||||
SystemConfigKey.UserInstalledPlugins
|
||||
) or []
|
||||
}
|
||||
for plugin_dir in PLUGIN_DIR.iterdir():
|
||||
if plugin_dir.is_dir():
|
||||
requirements_file = plugin_dir / "requirements.txt"
|
||||
if requirements_file.exists():
|
||||
if plugin_dir.name not in install_plugins:
|
||||
# 这个插件不在安装列表中 忽略它的依赖
|
||||
logger.debug(f"忽略插件 {plugin_dir.name} 的依赖")
|
||||
continue
|
||||
# 解析当前插件的 requirements.txt,获取依赖项
|
||||
plugin_deps = self.__parse_requirements(requirements_file)
|
||||
for pkg_name, version_specifiers in plugin_deps.items():
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
from enum import Enum
|
||||
from typing import Union, Dict, Optional
|
||||
from typing import Union, Optional
|
||||
|
||||
from app.schemas.types import ProgressKey
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.singleton import WeakSingleton
|
||||
|
||||
|
||||
class ProgressHelper(metaclass=Singleton):
|
||||
_process_detail: Dict[str, dict] = {}
|
||||
class ProgressHelper(metaclass=WeakSingleton):
|
||||
|
||||
def __init__(self):
|
||||
self._process_detail = {}
|
||||
|
||||
@@ -8,6 +8,7 @@ from app.log import logger
|
||||
from app.utils.http import RequestUtils
|
||||
from app.utils.string import StringUtils
|
||||
from app.utils.system import SystemUtils
|
||||
from version import APP_VERSION
|
||||
|
||||
|
||||
class ResourceHelper:
|
||||
@@ -58,15 +59,15 @@ class ResourceHelper:
|
||||
if rtype == "auth":
|
||||
# 站点认证资源
|
||||
local_version = SitesHelper().auth_version
|
||||
# 阻断v2.3.0以下的版本直接更新,避免无限重启
|
||||
# 阻断站点认证资源v2.3.0以下的版本直接更新,避免无限重启
|
||||
if StringUtils.compare_version(local_version, "<", "2.3.0"):
|
||||
continue
|
||||
# 阻断主程序版本v2.6.3以下的版本直接更新,避免搜索异常
|
||||
if StringUtils.compare_version(APP_VERSION, "<", "2.6.3"):
|
||||
continue
|
||||
elif rtype == "sites":
|
||||
# 站点索引资源
|
||||
local_version = SitesHelper().indexer_version
|
||||
# 阻断v2.0.0以下的版本直接更新,避免无限重启
|
||||
if StringUtils.compare_version(local_version, "<", "2.0.0"):
|
||||
continue
|
||||
else:
|
||||
continue
|
||||
if StringUtils.compare_version(version, ">", local_version):
|
||||
@@ -84,6 +85,8 @@ class ResourceHelper:
|
||||
elif not r:
|
||||
return None, "连接仓库失败"
|
||||
files_info = r.json()
|
||||
# 下载资源文件
|
||||
success = True
|
||||
for item in files_info:
|
||||
save_path = need_updates.get(item.get("name"))
|
||||
if not save_path:
|
||||
@@ -96,16 +99,23 @@ class ResourceHelper:
|
||||
timeout=180).get_res(download_url)
|
||||
if not res:
|
||||
logger.error(f"文件 {item.get('name')} 下载失败!")
|
||||
success = False
|
||||
break
|
||||
elif res.status_code != 200:
|
||||
logger.error(f"下载文件 {item.get('name')} 失败:{res.status_code} - {res.reason}")
|
||||
success = False
|
||||
break
|
||||
# 创建插件文件夹
|
||||
file_path = self._base_dir / save_path / item.get("name")
|
||||
if not file_path.parent.exists():
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# 写入文件
|
||||
file_path.write_bytes(res.content)
|
||||
logger.info("资源包更新完成,开始重启服务...")
|
||||
SystemHelper.restart()
|
||||
if success:
|
||||
logger.info("资源包更新完成,开始重启服务...")
|
||||
SystemHelper.restart()
|
||||
else:
|
||||
logger.warn("资源包更新失败,跳过升级!")
|
||||
else:
|
||||
logger.info("所有资源已最新,无需更新")
|
||||
except json.JSONDecodeError:
|
||||
|
||||
@@ -8,11 +8,11 @@ from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.log import logger
|
||||
from app.schemas.types import SystemConfigKey
|
||||
from app.utils.http import RequestUtils
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.singleton import WeakSingleton
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
|
||||
class SubscribeHelper(metaclass=Singleton):
|
||||
class SubscribeHelper(metaclass=WeakSingleton):
|
||||
"""
|
||||
订阅数据统计/订阅分享等
|
||||
"""
|
||||
@@ -29,6 +29,8 @@ class SubscribeHelper(metaclass=Singleton):
|
||||
|
||||
_sub_shares = f"{settings.MP_SERVER_HOST}/subscribe/shares"
|
||||
|
||||
_sub_share_statistic = f"{settings.MP_SERVER_HOST}/subscribe/share/statistics"
|
||||
|
||||
_sub_fork = f"{settings.MP_SERVER_HOST}/subscribe/fork/%s"
|
||||
|
||||
_shares_cache_region = "subscribe_share"
|
||||
@@ -199,7 +201,7 @@ class SubscribeHelper(metaclass=Singleton):
|
||||
else:
|
||||
return False, res.json().get("message")
|
||||
|
||||
@cached(region=_shares_cache_region)
|
||||
@cached(region=_shares_cache_region, maxsize=1, ttl=1800, skip_empty=True)
|
||||
def get_shares(self, name: Optional[str] = None, page: Optional[int] = 1, count: Optional[int] = 30) -> List[dict]:
|
||||
"""
|
||||
获取订阅分享数据
|
||||
@@ -215,6 +217,18 @@ class SubscribeHelper(metaclass=Singleton):
|
||||
return res.json()
|
||||
return []
|
||||
|
||||
@cached(region=_shares_cache_region, maxsize=1, ttl=1800, skip_empty=True)
|
||||
def get_share_statistics(self) -> List[dict]:
|
||||
"""
|
||||
获取订阅分享统计数据
|
||||
"""
|
||||
if not settings.SUBSCRIBE_STATISTIC_SHARE:
|
||||
return []
|
||||
res = RequestUtils(proxies=settings.PROXY, timeout=15).get_res(self._sub_share_statistic)
|
||||
if res and res.status_code == 200:
|
||||
return res.json()
|
||||
return []
|
||||
|
||||
def get_user_uuid(self) -> str:
|
||||
"""
|
||||
获取用户uuid
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
import datetime
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Tuple, Optional, List, Union, Dict
|
||||
from typing import Tuple, Optional, List, Union, Dict, Any
|
||||
from urllib.parse import unquote
|
||||
|
||||
from requests import Response
|
||||
from torrentool.api import Torrent
|
||||
|
||||
from app.core.config import settings
|
||||
@@ -16,17 +15,17 @@ from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.log import logger
|
||||
from app.schemas.types import MediaType, SystemConfigKey
|
||||
from app.utils.http import RequestUtils
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.singleton import WeakSingleton
|
||||
from app.utils.string import StringUtils
|
||||
|
||||
|
||||
class TorrentHelper(metaclass=Singleton):
|
||||
class TorrentHelper(metaclass=WeakSingleton):
|
||||
"""
|
||||
种子帮助类
|
||||
"""
|
||||
|
||||
# 失败的种子:站点链接
|
||||
_invalid_torrents = []
|
||||
def __init__(self):
|
||||
self._invalid_torrents = []
|
||||
|
||||
def download_torrent(self, url: str,
|
||||
cookie: Optional[str] = None,
|
||||
@@ -40,6 +39,22 @@ class TorrentHelper(metaclass=Singleton):
|
||||
"""
|
||||
if url.startswith("magnet:"):
|
||||
return None, url, "", [], f"磁力链接"
|
||||
# 构建 torrent 种子文件的存储路径
|
||||
file_path = (Path(settings.TEMP_PATH) / StringUtils.md5_hash(url)).with_suffix(".torrent")
|
||||
if file_path.exists():
|
||||
try:
|
||||
# 获取种子目录和文件清单
|
||||
folder_name, file_list = self.get_torrent_info(file_path)
|
||||
# 无法获取信息,则认为缓存文件无效
|
||||
if not folder_name and not file_list:
|
||||
raise ValueError("无效的缓存种子文件")
|
||||
# 获取种子数据
|
||||
content = file_path.read_bytes()
|
||||
# 成功拿到种子数据
|
||||
return file_path, content, folder_name, file_list, ""
|
||||
except Exception as err:
|
||||
logger.error(f"处理缓存的种子文件 {file_path} 时出错: {err},将重新下载")
|
||||
file_path.unlink(missing_ok=True)
|
||||
# 请求种子文件
|
||||
req = RequestUtils(
|
||||
ua=ua,
|
||||
@@ -106,10 +121,6 @@ class TorrentHelper(metaclass=Singleton):
|
||||
if req.content:
|
||||
# 检查是不是种子文件,如果不是仍然抛出异常
|
||||
try:
|
||||
# 读取种子文件名
|
||||
file_name = self.get_url_filename(req, url)
|
||||
# 种子文件路径
|
||||
file_path = Path(settings.TEMP_PATH) / file_name
|
||||
# 保存到文件
|
||||
file_path.write_bytes(req.content)
|
||||
# 获取种子目录和文件清单
|
||||
@@ -170,7 +181,7 @@ class TorrentHelper(metaclass=Singleton):
|
||||
return "", []
|
||||
|
||||
@staticmethod
|
||||
def get_url_filename(req: Response, url: str) -> str:
|
||||
def get_url_filename(req: Any, url: str) -> str:
|
||||
"""
|
||||
从下载请求中获取种子文件名
|
||||
"""
|
||||
@@ -308,7 +319,7 @@ class TorrentHelper(metaclass=Singleton):
|
||||
self._invalid_torrents.append(url)
|
||||
|
||||
@staticmethod
|
||||
def match_torrent(mediainfo: MediaInfo, torrent_meta: MetaInfo, torrent: TorrentInfo) -> bool:
|
||||
def match_torrent(mediainfo: MediaInfo, torrent_meta: MetaBase, torrent: TorrentInfo) -> bool:
|
||||
"""
|
||||
检查种子是否匹配媒体信息
|
||||
:param mediainfo: 需要匹配的媒体信息
|
||||
|
||||
132
app/helper/workflow.py
Normal file
132
app/helper/workflow.py
Normal file
@@ -0,0 +1,132 @@
|
||||
import json
|
||||
from typing import List, Tuple, Optional
|
||||
|
||||
from app.core.cache import cached, cache_backend
|
||||
from app.core.config import settings
|
||||
from app.db.workflow_oper import WorkflowOper
|
||||
from app.log import logger
|
||||
from app.utils.http import RequestUtils
|
||||
from app.utils.singleton import WeakSingleton
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
|
||||
class WorkflowHelper(metaclass=WeakSingleton):
|
||||
"""
|
||||
工作流分享等
|
||||
"""
|
||||
|
||||
_workflow_share = f"{settings.MP_SERVER_HOST}/workflow/share"
|
||||
|
||||
_workflow_shares = f"{settings.MP_SERVER_HOST}/workflow/shares"
|
||||
|
||||
_workflow_fork = f"{settings.MP_SERVER_HOST}/workflow/fork/%s"
|
||||
|
||||
_shares_cache_region = "workflow_share"
|
||||
|
||||
_share_user_id = None
|
||||
|
||||
def __init__(self):
|
||||
self.get_user_uuid()
|
||||
|
||||
def workflow_share(self, workflow_id: int,
|
||||
share_title: str, share_comment: str, share_user: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
分享工作流
|
||||
"""
|
||||
if not settings.WORKFLOW_STATISTIC_SHARE: # 使用独立的工作流分享开关
|
||||
return False, "当前没有开启工作流数据共享功能"
|
||||
|
||||
# 获取工作流信息
|
||||
workflow = WorkflowOper().get(workflow_id)
|
||||
if not workflow:
|
||||
return False, "工作流不存在"
|
||||
|
||||
if not workflow.actions or not workflow.flows:
|
||||
return False, "请分享有动作和流程的工作流"
|
||||
|
||||
workflow_dict = workflow.to_dict()
|
||||
workflow_dict.pop("id", None)
|
||||
workflow_dict.pop("context", None)
|
||||
workflow_dict['actions'] = json.dumps(workflow_dict['actions'] or [])
|
||||
workflow_dict['flows'] = json.dumps(workflow_dict['flows'] or [])
|
||||
|
||||
# 发送分享请求
|
||||
res = RequestUtils(proxies=settings.PROXY or {}, content_type="application/json",
|
||||
timeout=10).post(self._workflow_share,
|
||||
json={
|
||||
"share_title": share_title,
|
||||
"share_comment": share_comment,
|
||||
"share_user": share_user,
|
||||
"share_uid": self._share_user_id,
|
||||
**workflow_dict
|
||||
})
|
||||
if res is None:
|
||||
return False, "连接MoviePilot服务器失败"
|
||||
if res.ok:
|
||||
# 清除 get_shares 的缓存,以便实时看到结果
|
||||
cache_backend.clear(region=self._shares_cache_region)
|
||||
return True, ""
|
||||
else:
|
||||
return False, res.json().get("message")
|
||||
|
||||
def share_delete(self, share_id: int) -> Tuple[bool, str]:
|
||||
"""
|
||||
删除分享
|
||||
"""
|
||||
if not settings.WORKFLOW_STATISTIC_SHARE: # 使用独立的工作流分享开关
|
||||
return False, "当前没有开启工作流数据共享功能"
|
||||
|
||||
res = RequestUtils(proxies=settings.PROXY or {},
|
||||
timeout=5).delete_res(f"{self._workflow_share}/{share_id}",
|
||||
params={"share_uid": self._share_user_id})
|
||||
if res is None:
|
||||
return False, "连接MoviePilot服务器失败"
|
||||
if res.ok:
|
||||
# 清除 get_shares 的缓存,以便实时看到结果
|
||||
cache_backend.clear(region=self._shares_cache_region)
|
||||
return True, ""
|
||||
else:
|
||||
return False, res.json().get("message")
|
||||
|
||||
def workflow_fork(self, share_id: int) -> Tuple[bool, str]:
|
||||
"""
|
||||
复用分享的工作流
|
||||
"""
|
||||
if not settings.WORKFLOW_STATISTIC_SHARE: # 使用独立的工作流分享开关
|
||||
return False, "当前没有开启工作流数据共享功能"
|
||||
|
||||
res = RequestUtils(proxies=settings.PROXY or {}, timeout=5, headers={
|
||||
"Content-Type": "application/json"
|
||||
}).get_res(self._workflow_fork % share_id)
|
||||
if res is None:
|
||||
return False, "连接MoviePilot服务器失败"
|
||||
if res.ok:
|
||||
return True, ""
|
||||
else:
|
||||
return False, res.json().get("message")
|
||||
|
||||
@cached(region=_shares_cache_region, maxsize=1, skip_empty=True)
|
||||
def get_shares(self, name: Optional[str] = None, page: Optional[int] = 1, count: Optional[int] = 30) -> List[dict]:
|
||||
"""
|
||||
获取工作流分享数据
|
||||
"""
|
||||
if not settings.WORKFLOW_STATISTIC_SHARE: # 使用独立的工作流分享开关
|
||||
return []
|
||||
|
||||
res = RequestUtils(proxies=settings.PROXY or {}, timeout=15).get_res(self._workflow_shares, params={
|
||||
"name": name,
|
||||
"page": page,
|
||||
"count": count
|
||||
})
|
||||
if res and res.status_code == 200:
|
||||
return res.json()
|
||||
return []
|
||||
|
||||
def get_user_uuid(self) -> str:
|
||||
"""
|
||||
获取用户uuid
|
||||
"""
|
||||
if not self._share_user_id:
|
||||
self._share_user_id = SystemUtils.generate_user_unique_id()
|
||||
logger.info(f"当前用户UUID: {self._share_user_id}")
|
||||
return self._share_user_id or ""
|
||||
@@ -1,5 +1,6 @@
|
||||
import multiprocessing
|
||||
import os
|
||||
import setproctitle
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
@@ -19,6 +20,9 @@ if SystemUtils.is_frozen():
|
||||
from app.core.config import settings
|
||||
from app.db.init import init_db, update_db
|
||||
|
||||
# 设置进程名
|
||||
setproctitle.setproctitle(settings.PROJECT_NAME)
|
||||
|
||||
# uvicorn服务
|
||||
Server = uvicorn.Server(Config(app, host=settings.HOST, port=settings.PORT,
|
||||
reload=settings.DEV, workers=multiprocessing.cpu_count(),
|
||||
|
||||
@@ -12,10 +12,10 @@ import requests
|
||||
from app.core.cache import cached
|
||||
from app.core.config import settings
|
||||
from app.utils.http import RequestUtils
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.singleton import WeakSingleton
|
||||
|
||||
|
||||
class DoubanApi(metaclass=Singleton):
|
||||
class DoubanApi(metaclass=WeakSingleton):
|
||||
_urls = {
|
||||
# 搜索类
|
||||
# sort=U:近期热门 T:标记最多 S:评分最高 R:最新上映
|
||||
@@ -151,7 +151,6 @@ class DoubanApi(metaclass=Singleton):
|
||||
_api_key2 = "0ab215a8b1977939201640fa14c66bab"
|
||||
_base_url = "https://frodo.douban.com/api/v2"
|
||||
_api_url = "https://api.douban.com/v2"
|
||||
_session = None
|
||||
|
||||
def __init__(self):
|
||||
self._session = requests.Session()
|
||||
@@ -235,7 +234,7 @@ class DoubanApi(metaclass=Singleton):
|
||||
if '_ts' in params:
|
||||
params.pop('_ts')
|
||||
resp = RequestUtils(
|
||||
ua=settings.USER_AGENT,
|
||||
ua=settings.NORMAL_USER_AGENT,
|
||||
session=self._session,
|
||||
).post_res(url=req_url, data=params)
|
||||
if resp is not None and resp.status_code == 400 and "rate_limit" in resp.text:
|
||||
|
||||
@@ -10,7 +10,7 @@ from app.core.config import settings
|
||||
from app.core.meta import MetaBase
|
||||
from app.core.metainfo import MetaInfo
|
||||
from app.log import logger
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.singleton import WeakSingleton
|
||||
from app.schemas.types import MediaType
|
||||
|
||||
lock = RLock()
|
||||
@@ -19,7 +19,7 @@ CACHE_EXPIRE_TIMESTAMP_STR = "cache_expire_timestamp"
|
||||
EXPIRE_TIMESTAMP = settings.CONF.meta
|
||||
|
||||
|
||||
class DoubanCache(metaclass=Singleton):
|
||||
class DoubanCache(metaclass=WeakSingleton):
|
||||
"""
|
||||
豆瓣缓存数据
|
||||
{
|
||||
@@ -29,9 +29,6 @@ class DoubanCache(metaclass=Singleton):
|
||||
"type": MediaType
|
||||
}
|
||||
"""
|
||||
_meta_data: dict = {}
|
||||
# 缓存文件路径
|
||||
_meta_path: Path = None
|
||||
# TMDB缓存过期
|
||||
_tmdb_cache_expire: bool = True
|
||||
|
||||
@@ -233,3 +230,6 @@ class DoubanCache(metaclass=Singleton):
|
||||
if not cache_media_info:
|
||||
return
|
||||
self._meta_data[key]['title'] = cn_title
|
||||
|
||||
def __del__(self):
|
||||
self.save()
|
||||
|
||||
@@ -166,7 +166,8 @@ class Emby:
|
||||
type=library_type,
|
||||
image=image,
|
||||
link=f'{self._playhost or self._host}web/index.html'
|
||||
f'#!/videos?serverId={self.serverid}&parentId={library.get("Id")}'
|
||||
f'#!/videos?serverId={self.serverid}&parentId={library.get("Id")}',
|
||||
server_type= "emby"
|
||||
)
|
||||
)
|
||||
return libraries
|
||||
@@ -1167,7 +1168,8 @@ class Emby:
|
||||
type=item_type,
|
||||
image=image,
|
||||
link=link,
|
||||
percent=item.get("UserData", {}).get("PlayedPercentage")
|
||||
percent=item.get("UserData", {}).get("PlayedPercentage"),
|
||||
server_type='emby'
|
||||
))
|
||||
return ret_resume
|
||||
else:
|
||||
@@ -1219,7 +1221,8 @@ class Emby:
|
||||
type=item_type,
|
||||
image=image,
|
||||
link=link,
|
||||
BackdropImageTags=item.get("BackdropImageTags")
|
||||
BackdropImageTags=item.get("BackdropImageTags"),
|
||||
server_type='emby'
|
||||
))
|
||||
return ret_latest
|
||||
else:
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Tuple, Union, Dict, Callable
|
||||
|
||||
from app.chain.tmdb import TmdbChain
|
||||
from app.core.config import settings
|
||||
from app.core.context import MediaInfo
|
||||
from app.core.meta import MetaBase
|
||||
@@ -139,16 +140,32 @@ class FileManagerModule(_ModuleBase):
|
||||
"""
|
||||
handler = TransHandler()
|
||||
# 重命名格式
|
||||
rename_format = settings.TV_RENAME_FORMAT \
|
||||
if mediainfo.type == MediaType.TV else settings.MOVIE_RENAME_FORMAT
|
||||
rename_format = settings.RENAME_FORMAT(mediainfo.type)
|
||||
# 获取集信息
|
||||
episodes_info: Optional[List[TmdbEpisode]] = None
|
||||
if mediainfo.type == MediaType.TV:
|
||||
# 判断注意season为0的情况
|
||||
season_num = mediainfo.season
|
||||
if season_num is None and meta.season_seq:
|
||||
if meta.season_seq.isdigit():
|
||||
season_num = int(meta.season_seq)
|
||||
# 默认值1
|
||||
if season_num is None:
|
||||
season_num = 1
|
||||
episodes_info = TmdbChain().tmdb_episodes(
|
||||
tmdbid=mediainfo.tmdb_id,
|
||||
season=season_num,
|
||||
episode_group=mediainfo.episode_group,
|
||||
)
|
||||
# 获取重命名后的名称
|
||||
path = handler.get_rename_path(
|
||||
template_string=rename_format,
|
||||
rename_dict=handler.get_naming_dict(meta=meta,
|
||||
mediainfo=mediainfo,
|
||||
episodes_info=episodes_info,
|
||||
file_ext=Path(meta.title).suffix)
|
||||
)
|
||||
return str(path)
|
||||
return path.as_posix() if path else ""
|
||||
|
||||
def save_config(self, storage: str, conf: Dict) -> None:
|
||||
"""
|
||||
@@ -411,6 +428,12 @@ class FileManagerModule(_ModuleBase):
|
||||
message=f"{target_path} 不是有效目录")
|
||||
# 获取目标路径
|
||||
if target_directory:
|
||||
# 目标媒体库目录未设置
|
||||
if not target_directory.library_path:
|
||||
logger.error(f"目标媒体库目录未设置,无法整理文件,源路径:{fileitem.path}")
|
||||
return TransferInfo(success=False,
|
||||
fileitem=fileitem,
|
||||
message="目标媒体库目录未设置")
|
||||
# 整理方式
|
||||
if not transfer_type:
|
||||
transfer_type = target_directory.transfer_type
|
||||
@@ -510,19 +533,35 @@ class FileManagerModule(_ModuleBase):
|
||||
# 媒体分类路径
|
||||
dir_path = handler.get_dest_dir(mediainfo=mediainfo, target_dir=dest_dir)
|
||||
# 重命名格式
|
||||
rename_format = settings.TV_RENAME_FORMAT \
|
||||
if mediainfo.type == MediaType.TV else settings.MOVIE_RENAME_FORMAT
|
||||
rename_format = settings.RENAME_FORMAT(mediainfo.type)
|
||||
# 元数据补上常用属性,尽可能确保重命名后的路径不出现空白
|
||||
meta = MetaInfo(mediainfo.title)
|
||||
if meta.type == MediaType.UNKNOWN and mediainfo.type is not None:
|
||||
meta.type = mediainfo.type
|
||||
if meta.year is None:
|
||||
meta.year = mediainfo.year
|
||||
if meta.begin_season is None:
|
||||
meta.begin_season = 1
|
||||
if meta.begin_episode is None:
|
||||
meta.begin_episode = 1
|
||||
# 获取路径(重命名路径)
|
||||
target_path = handler.get_rename_path(
|
||||
path=dir_path,
|
||||
template_string=rename_format,
|
||||
rename_dict=handler.get_naming_dict(meta=MetaInfo(mediainfo.title),
|
||||
rename_dict=handler.get_naming_dict(meta=meta,
|
||||
mediainfo=mediainfo)
|
||||
)
|
||||
# 计算重命名中的文件夹层数
|
||||
rename_format_level = len(rename_format.split("/")) - 1
|
||||
# 取相对路径的第1层目录
|
||||
media_path = target_path.parents[rename_format_level - 1]
|
||||
# 获取重命名后的媒体文件根路径
|
||||
media_path = DirectoryHelper.get_media_root_path(
|
||||
rename_format, rename_path=target_path
|
||||
)
|
||||
if not media_path:
|
||||
# 忽略
|
||||
continue
|
||||
if dir_path != media_path and dir_path.is_relative_to(media_path):
|
||||
# 兜底检查,避免不必要的扫盘
|
||||
logger.warn(f"{media_path} 是媒体库目录 {dir_path} 的父目录,忽略获取媒体文件列表,请检查重命名格式!")
|
||||
continue
|
||||
# 检索媒体文件
|
||||
fileitem = storage_oper.get_item(media_path)
|
||||
if not fileitem:
|
||||
@@ -548,9 +587,12 @@ class FileManagerModule(_ModuleBase):
|
||||
if not settings.LOCAL_EXISTS_SEARCH:
|
||||
return None
|
||||
|
||||
logger.debug(f"正在本地媒体库中查找 {mediainfo.title_year}...")
|
||||
|
||||
# 检查媒体库
|
||||
fileitems = self.media_files(mediainfo)
|
||||
if not fileitems:
|
||||
logger.debug(f"{mediainfo.title_year} 不在本地媒体库中")
|
||||
return None
|
||||
|
||||
if mediainfo.type == MediaType.MOVIE:
|
||||
|
||||
@@ -13,6 +13,7 @@ class StorageBase(metaclass=ABCMeta):
|
||||
"""
|
||||
schema = None
|
||||
transtype = {}
|
||||
snapshot_check_folder_modtime = True
|
||||
|
||||
def __init__(self):
|
||||
self.storagehelper = StorageHelper()
|
||||
@@ -214,7 +215,8 @@ class StorageBase(metaclass=ABCMeta):
|
||||
return
|
||||
|
||||
# 增量检查:如果目录修改时间早于上次快照,跳过
|
||||
if (last_snapshot_time and
|
||||
if (self.snapshot_check_folder_modtime and
|
||||
last_snapshot_time and
|
||||
_fileitm.modify_time and
|
||||
_fileitm.modify_time <= last_snapshot_time):
|
||||
return
|
||||
@@ -225,11 +227,13 @@ class StorageBase(metaclass=ABCMeta):
|
||||
__snapshot_file(sub_file, current_depth + 1)
|
||||
else:
|
||||
# 记录文件的完整信息用于比对
|
||||
files_info[_fileitm.path] = {
|
||||
'size': _fileitm.size or 0,
|
||||
'modify_time': getattr(_fileitm, 'modify_time', 0),
|
||||
'type': _fileitm.type
|
||||
}
|
||||
if getattr(_fileitm, 'modify_time', 0) > last_snapshot_time:
|
||||
files_info[_fileitm.path] = {
|
||||
'size': _fileitm.size or 0,
|
||||
'modify_time': getattr(_fileitm, 'modify_time', 0),
|
||||
'type': _fileitm.type
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Snapshot error for {_fileitm.path}: {e}")
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ from app.core.config import settings
|
||||
from app.log import logger
|
||||
from app.modules.filemanager import StorageBase
|
||||
from app.schemas.types import StorageSchema
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.singleton import WeakSingleton
|
||||
from app.utils.string import StringUtils
|
||||
|
||||
lock = threading.Lock()
|
||||
@@ -29,7 +29,7 @@ class SessionInvalidException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class AliPan(StorageBase, metaclass=Singleton):
|
||||
class AliPan(StorageBase, metaclass=WeakSingleton):
|
||||
"""
|
||||
阿里云盘相关操作
|
||||
"""
|
||||
@@ -43,17 +43,12 @@ class AliPan(StorageBase, metaclass=Singleton):
|
||||
"copy": "复制"
|
||||
}
|
||||
|
||||
# 验证参数
|
||||
_auth_state = {}
|
||||
|
||||
# 上传进度值
|
||||
_last_progress = 0
|
||||
|
||||
# 基础url
|
||||
base_url = "https://openapi.alipan.com"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._auth_state = {}
|
||||
self.session = requests.Session()
|
||||
self._init_session()
|
||||
|
||||
@@ -244,6 +239,7 @@ class AliPan(StorageBase, metaclass=Singleton):
|
||||
conf = self.get_conf()
|
||||
conf.update(result)
|
||||
self.set_config(conf)
|
||||
return None
|
||||
|
||||
def _request_api(self, method: str, endpoint: str,
|
||||
result_key: Optional[str] = None, **kwargs) -> Optional[Union[dict, list]]:
|
||||
@@ -369,7 +365,7 @@ class AliPan(StorageBase, metaclass=Singleton):
|
||||
break
|
||||
next_marker = resp.get("next_marker")
|
||||
for item in resp.get("items", []):
|
||||
items.append(self.__get_fileitem(item, parent=fileitem.path))
|
||||
items.append(self.__get_fileitem(item, parent=str(fileitem.path)))
|
||||
if len(resp.get("items")) < 100:
|
||||
break
|
||||
return items
|
||||
@@ -787,7 +783,7 @@ class AliPan(StorageBase, metaclass=Singleton):
|
||||
"/adrive/v1.0/openFile/get_by_path",
|
||||
json={
|
||||
"drive_id": drive_id or self._default_drive_id,
|
||||
"file_path": str(path)
|
||||
"file_path": path.as_posix()
|
||||
}
|
||||
)
|
||||
if not resp:
|
||||
|
||||
@@ -12,11 +12,11 @@ from app.log import logger
|
||||
from app.modules.filemanager.storages import StorageBase
|
||||
from app.schemas.types import StorageSchema
|
||||
from app.utils.http import RequestUtils
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.singleton import WeakSingleton
|
||||
from app.utils.url import UrlUtils
|
||||
|
||||
|
||||
class Alist(StorageBase, metaclass=Singleton):
|
||||
class Alist(StorageBase, metaclass=WeakSingleton):
|
||||
"""
|
||||
Alist相关操作
|
||||
api文档:https://oplist.org/zh/
|
||||
@@ -31,6 +31,8 @@ class Alist(StorageBase, metaclass=Singleton):
|
||||
"move": "移动",
|
||||
}
|
||||
|
||||
snapshot_check_folder_modtime = settings.OPENLIST_SNAPSHOT_CHECK_FOLDER_MODTIME
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
@@ -38,7 +40,7 @@ class Alist(StorageBase, metaclass=Singleton):
|
||||
"""
|
||||
初始化
|
||||
"""
|
||||
self.__generate_token.clear_cache()
|
||||
self.__generate_token.clear_cache() # noqa
|
||||
|
||||
@property
|
||||
def __get_base_url(self) -> str:
|
||||
@@ -376,10 +378,46 @@ class Alist(StorageBase, metaclass=Singleton):
|
||||
"""
|
||||
return self.get_folder(Path(fileitem.path).parent)
|
||||
|
||||
def __is_empty_dir(self, fileitem: schemas.FileItem) -> bool:
|
||||
"""
|
||||
判断目录是否为空
|
||||
"""
|
||||
if fileitem.type != "dir":
|
||||
return False
|
||||
# 获取目录内容
|
||||
items = self.list(fileitem)
|
||||
return len(items) == 0
|
||||
|
||||
def delete(self, fileitem: schemas.FileItem) -> bool:
|
||||
"""
|
||||
删除文件
|
||||
删除文件或目录,空目录用专用API
|
||||
"""
|
||||
# 如果是空目录,优先用 remove_empty_directory
|
||||
if fileitem.type == "dir" and self.__is_empty_dir(fileitem):
|
||||
resp = RequestUtils(
|
||||
headers=self.__get_header_with_token()
|
||||
).post_res(
|
||||
self.__get_api_url("/api/fs/remove_empty_directory"),
|
||||
json={
|
||||
"src_dir": fileitem.path,
|
||||
},
|
||||
)
|
||||
if resp is None:
|
||||
logger.warn(f"【OpenList】请求删除空目录 {fileitem.path} 失败,无法连接alist服务")
|
||||
return False
|
||||
if resp.status_code != 200:
|
||||
logger.warn(
|
||||
f"【OpenList】请求删除空目录 {fileitem.path} 失败,状态码:{resp.status_code}"
|
||||
)
|
||||
return False
|
||||
result = resp.json()
|
||||
if result["code"] != 200:
|
||||
logger.warn(
|
||||
f'【OpenList】删除空目录 {fileitem.path} 失败,错误信息:{result["message"]}'
|
||||
)
|
||||
return False
|
||||
return True
|
||||
# 其它情况(文件或非空目录)
|
||||
resp = RequestUtils(
|
||||
headers=self.__get_header_with_token()
|
||||
).post_res(
|
||||
@@ -389,20 +427,6 @@ class Alist(StorageBase, metaclass=Singleton):
|
||||
"names": [fileitem.name],
|
||||
},
|
||||
)
|
||||
"""
|
||||
{
|
||||
"names": [
|
||||
"string"
|
||||
],
|
||||
"dir": "string"
|
||||
}
|
||||
======================================
|
||||
{
|
||||
"code": 200,
|
||||
"message": "success",
|
||||
"data": null
|
||||
}
|
||||
"""
|
||||
if resp is None:
|
||||
logger.warn(f"【OpenList】请求删除文件 {fileitem.path} 失败,无法连接alist服务")
|
||||
return False
|
||||
@@ -411,7 +435,6 @@ class Alist(StorageBase, metaclass=Singleton):
|
||||
f"【OpenList】请求删除文件 {fileitem.path} 失败,状态码:{resp.status_code}"
|
||||
)
|
||||
return False
|
||||
|
||||
result = resp.json()
|
||||
if result["code"] != 200:
|
||||
logger.warn(
|
||||
@@ -565,6 +588,9 @@ class Alist(StorageBase, metaclass=Singleton):
|
||||
data=f,
|
||||
)
|
||||
|
||||
if resp is None:
|
||||
logger.warn(f"【OpenList】请求上传文件 {path} 失败")
|
||||
return None
|
||||
if resp.status_code != 200:
|
||||
logger.warn(f"【OpenList】请求上传文件 {path} 失败,状态码:{resp.status_code}")
|
||||
return None
|
||||
|
||||
@@ -44,7 +44,7 @@ class LocalStorage(StorageBase):
|
||||
return schemas.FileItem(
|
||||
storage=self.schema.value,
|
||||
type="file",
|
||||
path=str(path).replace("\\", "/"),
|
||||
path=path.as_posix(),
|
||||
name=path.name,
|
||||
basename=path.stem,
|
||||
extension=path.suffix[1:],
|
||||
@@ -59,7 +59,7 @@ class LocalStorage(StorageBase):
|
||||
return schemas.FileItem(
|
||||
storage=self.schema.value,
|
||||
type="dir",
|
||||
path=str(path).replace("\\", "/") + "/",
|
||||
path=path.as_posix() + "/",
|
||||
name=path.name,
|
||||
basename=path.stem,
|
||||
modify_time=path.stat().st_mtime,
|
||||
|
||||
@@ -26,6 +26,8 @@ class Rclone(StorageBase):
|
||||
"copy": "复制"
|
||||
}
|
||||
|
||||
snapshot_check_folder_modtime = settings.RCLONE_SNAPSHOT_CHECK_FOLDER_MODTIME
|
||||
|
||||
def init_storage(self):
|
||||
"""
|
||||
初始化
|
||||
@@ -267,7 +269,7 @@ class Rclone(StorageBase):
|
||||
retcode = subprocess.run(
|
||||
[
|
||||
'rclone', 'copyto',
|
||||
str(path),
|
||||
path.as_posix(),
|
||||
f'MP:{new_path}'
|
||||
],
|
||||
startupinfo=self.__get_hidden_shell()
|
||||
|
||||
@@ -12,17 +12,19 @@ from app.core.config import settings
|
||||
from app.log import logger
|
||||
from app.modules.filemanager import StorageBase
|
||||
from app.schemas.types import StorageSchema
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.singleton import WeakSingleton
|
||||
|
||||
lock = threading.Lock()
|
||||
|
||||
|
||||
class SMBConnectionError(Exception):
|
||||
"""SMB 连接错误"""
|
||||
"""
|
||||
SMB 连接错误
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class SMB(StorageBase, metaclass=Singleton):
|
||||
class SMB(StorageBase, metaclass=WeakSingleton):
|
||||
"""
|
||||
SMB网络挂载存储相关操作 - 使用 smbclient 高级接口
|
||||
"""
|
||||
|
||||
@@ -18,7 +18,7 @@ from app.core.config import settings
|
||||
from app.log import logger
|
||||
from app.modules.filemanager import StorageBase
|
||||
from app.schemas.types import StorageSchema
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.singleton import WeakSingleton
|
||||
from app.utils.string import StringUtils
|
||||
|
||||
lock = threading.Lock()
|
||||
@@ -28,7 +28,7 @@ class NoCheckInException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class U115Pan(StorageBase, metaclass=Singleton):
|
||||
class U115Pan(StorageBase, metaclass=WeakSingleton):
|
||||
"""
|
||||
115相关操作
|
||||
"""
|
||||
@@ -41,18 +41,12 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
||||
"move": "移动",
|
||||
"copy": "复制"
|
||||
}
|
||||
|
||||
# 验证参数
|
||||
_auth_state = {}
|
||||
|
||||
# 上传进度值
|
||||
_last_progress = 0
|
||||
|
||||
# 基础url
|
||||
base_url = "https://proapi.115.com"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._auth_state = {}
|
||||
self.session = requests.Session()
|
||||
self._init_session()
|
||||
|
||||
@@ -219,8 +213,11 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
||||
|
||||
# 处理速率限制
|
||||
if resp.status_code == 429:
|
||||
reset_time = int(resp.headers.get("X-RateLimit-Reset", 60))
|
||||
time.sleep(reset_time + 5)
|
||||
reset_time = 5 + int(resp.headers.get("X-RateLimit-Reset", 60))
|
||||
logger.debug(
|
||||
f"【115】{method} 请求 {endpoint} 限流,等待{reset_time}秒后重试"
|
||||
)
|
||||
time.sleep(reset_time)
|
||||
return self._request_api(method, endpoint, result_key, **kwargs)
|
||||
|
||||
# 处理请求错误
|
||||
@@ -489,7 +486,8 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
||||
type="file" if info_resp["file_category"] == "1" else "dir",
|
||||
name=info_resp["file_name"],
|
||||
basename=Path(info_resp["file_name"]).stem,
|
||||
extension=Path(info_resp["file_name"]).suffix[1:] if info_resp["file_category"] == "1" else None,
|
||||
extension=Path(info_resp["file_name"]).suffix[1:] if info_resp[
|
||||
"file_category"] == "1" else None,
|
||||
pickcode=info_resp["pick_code"],
|
||||
size=StringUtils.num_filesize(info_resp['size']) if info_resp["file_category"] == "1" else None,
|
||||
modify_time=info_resp["utime"]
|
||||
@@ -674,7 +672,7 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
||||
"/open/folder/get_info",
|
||||
"data",
|
||||
data={
|
||||
"path": str(path)
|
||||
"path": path.as_posix()
|
||||
}
|
||||
)
|
||||
if not resp:
|
||||
@@ -682,7 +680,7 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
||||
return schemas.FileItem(
|
||||
storage=self.schema.value,
|
||||
fileid=str(resp["file_id"]),
|
||||
path=str(path) + ("/" if resp["file_category"] == "0" else ""),
|
||||
path=path.as_posix() + ("/" if resp["file_category"] == "0" else ""),
|
||||
type="file" if resp["file_category"] == "1" else "dir",
|
||||
name=resp["file_name"],
|
||||
basename=Path(resp["file_name"]).stem,
|
||||
@@ -739,7 +737,11 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
||||
"""
|
||||
企业级复制实现(支持目录递归复制)
|
||||
"""
|
||||
src_fid = fileitem.fileid
|
||||
if fileitem.fileid is None:
|
||||
fileitem = self.get_item(Path(fileitem.path))
|
||||
if not fileitem:
|
||||
logger.warn(f"【115】获取文件 {fileitem.path} 失败!")
|
||||
return False
|
||||
dest_fileitem = self.get_item(path)
|
||||
if not dest_fileitem or dest_fileitem.type != "dir":
|
||||
logger.warn(f"【115】目标路径 {path} 不是一个有效的目录!")
|
||||
@@ -749,7 +751,7 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
||||
"POST",
|
||||
"/open/ufile/copy",
|
||||
data={
|
||||
"file_id": int(src_fid),
|
||||
"file_id": int(fileitem.fileid),
|
||||
"pid": int(dest_fileitem.fileid),
|
||||
}
|
||||
)
|
||||
@@ -766,7 +768,11 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
||||
"""
|
||||
原子性移动操作实现
|
||||
"""
|
||||
src_fid = fileitem.fileid
|
||||
if fileitem.fileid is None:
|
||||
fileitem = self.get_item(Path(fileitem.path))
|
||||
if not fileitem:
|
||||
logger.warn(f"【115】获取文件 {fileitem.path} 失败!")
|
||||
return False
|
||||
dest_fileitem = self.get_item(path)
|
||||
if not dest_fileitem or dest_fileitem.type != "dir":
|
||||
logger.warn(f"【115】目标路径 {path} 不是一个有效的目录!")
|
||||
@@ -775,7 +781,7 @@ class U115Pan(StorageBase, metaclass=Singleton):
|
||||
"POST",
|
||||
"/open/ufile/move",
|
||||
data={
|
||||
"file_ids": int(src_fid),
|
||||
"file_ids": int(fileitem.fileid),
|
||||
"to_cid": int(dest_fileitem.fileid),
|
||||
}
|
||||
)
|
||||
|
||||
@@ -10,6 +10,7 @@ from app.core.context import MediaInfo
|
||||
from app.core.event import eventmanager
|
||||
from app.core.meta import MetaBase
|
||||
from app.core.metainfo import MetaInfoPath
|
||||
from app.helper.directory import DirectoryHelper
|
||||
from app.helper.message import TemplateHelper
|
||||
from app.log import logger
|
||||
from app.modules.filemanager.storages import StorageBase
|
||||
@@ -26,11 +27,10 @@ class TransHandler:
|
||||
文件转移整理类
|
||||
"""
|
||||
|
||||
result: Optional[TransferInfo] = None
|
||||
inner_lock: Lock = Lock()
|
||||
|
||||
def __init__(self):
|
||||
self.__reset_result()
|
||||
self.result = None
|
||||
|
||||
def __reset_result(self):
|
||||
"""
|
||||
@@ -103,185 +103,211 @@ class TransHandler:
|
||||
# 重置结果
|
||||
self.__reset_result()
|
||||
|
||||
# 重命名格式
|
||||
rename_format = settings.TV_RENAME_FORMAT \
|
||||
if mediainfo.type == MediaType.TV else settings.MOVIE_RENAME_FORMAT
|
||||
try:
|
||||
|
||||
# 判断是否为文件夹
|
||||
if fileitem.type == "dir":
|
||||
# 整理整个目录,一般为蓝光原盘
|
||||
if need_rename:
|
||||
new_path = self.get_rename_path(
|
||||
path=target_path,
|
||||
template_string=rename_format,
|
||||
rename_dict=self.get_naming_dict(meta=in_meta,
|
||||
mediainfo=mediainfo)
|
||||
).parent
|
||||
else:
|
||||
new_path = target_path / fileitem.name
|
||||
# 整理目录
|
||||
new_diritem, errmsg = self.__transfer_dir(fileitem=fileitem,
|
||||
mediainfo=mediainfo,
|
||||
source_oper=source_oper,
|
||||
target_oper=target_oper,
|
||||
target_storage=target_storage,
|
||||
target_path=new_path,
|
||||
transfer_type=transfer_type)
|
||||
if not new_diritem:
|
||||
logger.error(f"文件夹 {fileitem.path} 整理失败:{errmsg}")
|
||||
self.__set_result(success=False,
|
||||
message=errmsg,
|
||||
fileitem=fileitem,
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify)
|
||||
return self.result
|
||||
# 重命名格式
|
||||
rename_format = settings.RENAME_FORMAT(mediainfo.type)
|
||||
|
||||
logger.info(f"文件夹 {fileitem.path} 整理成功")
|
||||
# 计算目录下所有文件大小
|
||||
total_size = sum(file.stat().st_size for file in Path(fileitem.path).rglob('*') if file.is_file())
|
||||
# 返回整理后的路径
|
||||
self.__set_result(success=True,
|
||||
fileitem=fileitem,
|
||||
target_item=new_diritem,
|
||||
target_diritem=new_diritem,
|
||||
total_size=total_size,
|
||||
need_scrape=need_scrape,
|
||||
need_notify=need_notify,
|
||||
transfer_type=transfer_type)
|
||||
return self.result
|
||||
else:
|
||||
# 整理单个文件
|
||||
if mediainfo.type == MediaType.TV:
|
||||
# 电视剧
|
||||
if in_meta.begin_episode is None:
|
||||
logger.warn(f"文件 {fileitem.path} 整理失败:未识别到文件集数")
|
||||
# 判断是否为文件夹
|
||||
if fileitem.type == "dir":
|
||||
# 整理整个目录,一般为蓝光原盘
|
||||
if need_rename:
|
||||
new_path = self.get_rename_path(
|
||||
path=target_path,
|
||||
template_string=rename_format,
|
||||
rename_dict=self.get_naming_dict(meta=in_meta,
|
||||
mediainfo=mediainfo)
|
||||
)
|
||||
new_path = DirectoryHelper.get_media_root_path(
|
||||
rename_format, rename_path=new_path
|
||||
)
|
||||
if not new_path:
|
||||
self.__set_result(
|
||||
success=False,
|
||||
message="重命名格式无效",
|
||||
fileitem=fileitem,
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify,
|
||||
)
|
||||
return self.result.copy()
|
||||
else:
|
||||
new_path = target_path / fileitem.name
|
||||
# 整理目录
|
||||
new_diritem, errmsg = self.__transfer_dir(fileitem=fileitem,
|
||||
mediainfo=mediainfo,
|
||||
source_oper=source_oper,
|
||||
target_oper=target_oper,
|
||||
target_storage=target_storage,
|
||||
target_path=new_path,
|
||||
transfer_type=transfer_type)
|
||||
if not new_diritem:
|
||||
logger.error(f"文件夹 {fileitem.path} 整理失败:{errmsg}")
|
||||
self.__set_result(success=False,
|
||||
message=f"未识别到文件集数",
|
||||
message=errmsg,
|
||||
fileitem=fileitem,
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify)
|
||||
return self.result.copy()
|
||||
|
||||
logger.info(f"文件夹 {fileitem.path} 整理成功")
|
||||
# 计算目录下所有文件大小
|
||||
total_size = sum(file.stat().st_size for file in Path(fileitem.path).rglob('*') if file.is_file())
|
||||
# 返回整理后的路径
|
||||
self.__set_result(success=True,
|
||||
fileitem=fileitem,
|
||||
target_item=new_diritem,
|
||||
target_diritem=new_diritem,
|
||||
total_size=total_size,
|
||||
need_scrape=need_scrape,
|
||||
need_notify=need_notify,
|
||||
transfer_type=transfer_type)
|
||||
return self.result.copy()
|
||||
else:
|
||||
# 整理单个文件
|
||||
if mediainfo.type == MediaType.TV:
|
||||
# 电视剧
|
||||
if in_meta.begin_episode is None:
|
||||
logger.warn(f"文件 {fileitem.path} 整理失败:未识别到文件集数")
|
||||
self.__set_result(success=False,
|
||||
message="未识别到文件集数",
|
||||
fileitem=fileitem,
|
||||
fail_list=[fileitem.path],
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify)
|
||||
return self.result.copy()
|
||||
|
||||
# 文件结束季为空
|
||||
in_meta.end_season = None
|
||||
# 文件总季数为1
|
||||
if in_meta.total_season:
|
||||
in_meta.total_season = 1
|
||||
# 文件不可能超过2集
|
||||
if in_meta.total_episode > 2:
|
||||
in_meta.total_episode = 1
|
||||
in_meta.end_episode = None
|
||||
|
||||
# 目的文件名
|
||||
if need_rename:
|
||||
new_file = self.get_rename_path(
|
||||
path=target_path,
|
||||
template_string=rename_format,
|
||||
rename_dict=self.get_naming_dict(
|
||||
meta=in_meta,
|
||||
mediainfo=mediainfo,
|
||||
episodes_info=episodes_info,
|
||||
file_ext=f".{fileitem.extension}"
|
||||
)
|
||||
)
|
||||
folder_path = DirectoryHelper.get_media_root_path(
|
||||
rename_format, rename_path=new_file
|
||||
)
|
||||
if not folder_path:
|
||||
self.__set_result(
|
||||
success=False,
|
||||
message="重命名格式无效",
|
||||
fileitem=fileitem,
|
||||
fail_list=[fileitem.path],
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify,
|
||||
)
|
||||
return self.result.copy()
|
||||
else:
|
||||
new_file = target_path / fileitem.name
|
||||
folder_path = target_path
|
||||
|
||||
# 判断是否要覆盖
|
||||
overflag = False
|
||||
# 目标目录
|
||||
target_diritem = target_oper.get_folder(folder_path)
|
||||
if not target_diritem:
|
||||
logger.error(f"目标目录 {folder_path} 获取失败")
|
||||
self.__set_result(success=False,
|
||||
message=f"目标目录 {folder_path} 获取失败",
|
||||
fileitem=fileitem,
|
||||
fail_list=[fileitem.path],
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify)
|
||||
return self.result
|
||||
|
||||
# 文件结束季为空
|
||||
in_meta.end_season = None
|
||||
# 文件总季数为1
|
||||
if in_meta.total_season:
|
||||
in_meta.total_season = 1
|
||||
# 文件不可能超过2集
|
||||
if in_meta.total_episode > 2:
|
||||
in_meta.total_episode = 1
|
||||
in_meta.end_episode = None
|
||||
|
||||
# 目的文件名
|
||||
if need_rename:
|
||||
new_file = self.get_rename_path(
|
||||
path=target_path,
|
||||
template_string=rename_format,
|
||||
rename_dict=self.get_naming_dict(
|
||||
meta=in_meta,
|
||||
mediainfo=mediainfo,
|
||||
episodes_info=episodes_info,
|
||||
file_ext=f".{fileitem.extension}"
|
||||
)
|
||||
)
|
||||
else:
|
||||
new_file = target_path / fileitem.name
|
||||
|
||||
# 判断是否要覆盖
|
||||
overflag = False
|
||||
# 计算重命名中的文件夹层级
|
||||
rename_format_level = len(rename_format.split("/")) - 1
|
||||
folder_path = new_file.parents[rename_format_level - 1]
|
||||
# 目标目录
|
||||
target_diritem = target_oper.get_folder(folder_path)
|
||||
if not target_diritem:
|
||||
logger.error(f"目标目录 {folder_path} 获取失败")
|
||||
self.__set_result(success=False,
|
||||
message=f"目标目录 {folder_path} 获取失败",
|
||||
fileitem=fileitem,
|
||||
fail_list=[fileitem.path],
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify)
|
||||
return self.result
|
||||
# 目标文件
|
||||
target_item = target_oper.get_item(new_file)
|
||||
if target_item:
|
||||
# 目标文件已存在
|
||||
target_file = new_file
|
||||
if target_storage == "local" and new_file.is_symlink():
|
||||
target_file = new_file.readlink()
|
||||
if not target_file.exists():
|
||||
overflag = True
|
||||
if not overflag:
|
||||
return self.result.copy()
|
||||
# 目标文件
|
||||
target_item = target_oper.get_item(new_file)
|
||||
if target_item:
|
||||
# 目标文件已存在
|
||||
logger.info(f"目的文件系统中已经存在同名文件 {target_file},当前整理覆盖模式设置为 {overwrite_mode}")
|
||||
if overwrite_mode == 'always':
|
||||
# 总是覆盖同名文件
|
||||
overflag = True
|
||||
elif overwrite_mode == 'size':
|
||||
# 存在时大覆盖小
|
||||
if target_item.size < fileitem.size:
|
||||
logger.info(f"目标文件文件大小更小,将覆盖:{new_file}")
|
||||
target_file = new_file
|
||||
if target_storage == "local" and new_file.is_symlink():
|
||||
target_file = new_file.readlink()
|
||||
if not target_file.exists():
|
||||
overflag = True
|
||||
else:
|
||||
if not overflag:
|
||||
# 目标文件已存在
|
||||
logger.info(f"目的文件系统中已经存在同名文件 {target_file},当前整理覆盖模式设置为 {overwrite_mode}")
|
||||
if overwrite_mode == 'always':
|
||||
# 总是覆盖同名文件
|
||||
overflag = True
|
||||
elif overwrite_mode == 'size':
|
||||
# 存在时大覆盖小
|
||||
if target_item.size < fileitem.size:
|
||||
logger.info(f"目标文件文件大小更小,将覆盖:{new_file}")
|
||||
overflag = True
|
||||
else:
|
||||
self.__set_result(success=False,
|
||||
message=f"媒体库存在同名文件,且质量更好",
|
||||
fileitem=fileitem,
|
||||
target_item=target_item,
|
||||
target_diritem=target_diritem,
|
||||
fail_list=[fileitem.path],
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify)
|
||||
return self.result.copy()
|
||||
elif overwrite_mode == 'never':
|
||||
# 存在不覆盖
|
||||
self.__set_result(success=False,
|
||||
message=f"媒体库存在同名文件,且质量更好",
|
||||
message=f"媒体库存在同名文件,当前覆盖模式为不覆盖",
|
||||
fileitem=fileitem,
|
||||
target_item=target_item,
|
||||
target_diritem=target_diritem,
|
||||
fail_list=[fileitem.path],
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify)
|
||||
return self.result
|
||||
elif overwrite_mode == 'never':
|
||||
# 存在不覆盖
|
||||
self.__set_result(success=False,
|
||||
message=f"媒体库存在同名文件,当前覆盖模式为不覆盖",
|
||||
fileitem=fileitem,
|
||||
target_item=target_item,
|
||||
target_diritem=target_diritem,
|
||||
fail_list=[fileitem.path],
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify)
|
||||
return self.result
|
||||
elif overwrite_mode == 'latest':
|
||||
# 仅保留最新版本
|
||||
logger.info(f"当前整理覆盖模式设置为仅保留最新版本,将覆盖:{new_file}")
|
||||
overflag = True
|
||||
else:
|
||||
if overwrite_mode == 'latest':
|
||||
# 文件不存在,但仅保留最新版本
|
||||
logger.info(f"当前整理覆盖模式设置为 {overwrite_mode},仅保留最新版本,正在删除已有版本文件 ...")
|
||||
self.__delete_version_files(target_oper, new_file)
|
||||
# 整理文件
|
||||
new_item, err_msg = self.__transfer_file(fileitem=fileitem,
|
||||
mediainfo=mediainfo,
|
||||
target_storage=target_storage,
|
||||
target_file=new_file,
|
||||
transfer_type=transfer_type,
|
||||
over_flag=overflag,
|
||||
source_oper=source_oper,
|
||||
target_oper=target_oper)
|
||||
if not new_item:
|
||||
logger.error(f"文件 {fileitem.path} 整理失败:{err_msg}")
|
||||
self.__set_result(success=False,
|
||||
message=err_msg,
|
||||
return self.result.copy()
|
||||
elif overwrite_mode == 'latest':
|
||||
# 仅保留最新版本
|
||||
logger.info(f"当前整理覆盖模式设置为仅保留最新版本,将覆盖:{new_file}")
|
||||
overflag = True
|
||||
else:
|
||||
if overwrite_mode == 'latest':
|
||||
# 文件不存在,但仅保留最新版本
|
||||
logger.info(f"当前整理覆盖模式设置为 {overwrite_mode},仅保留最新版本,正在删除已有版本文件 ...")
|
||||
self.__delete_version_files(target_oper, new_file)
|
||||
# 整理文件
|
||||
new_item, err_msg = self.__transfer_file(fileitem=fileitem,
|
||||
mediainfo=mediainfo,
|
||||
target_storage=target_storage,
|
||||
target_file=new_file,
|
||||
transfer_type=transfer_type,
|
||||
over_flag=overflag,
|
||||
source_oper=source_oper,
|
||||
target_oper=target_oper)
|
||||
if not new_item:
|
||||
logger.error(f"文件 {fileitem.path} 整理失败:{err_msg}")
|
||||
self.__set_result(success=False,
|
||||
message=err_msg,
|
||||
fileitem=fileitem,
|
||||
fail_list=[fileitem.path],
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify)
|
||||
return self.result.copy()
|
||||
|
||||
logger.info(f"文件 {fileitem.path} 整理成功")
|
||||
self.__set_result(success=True,
|
||||
fileitem=fileitem,
|
||||
fail_list=[fileitem.path],
|
||||
target_item=new_item,
|
||||
target_diritem=target_diritem,
|
||||
need_scrape=need_scrape,
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify)
|
||||
return self.result
|
||||
|
||||
logger.info(f"文件 {fileitem.path} 整理成功")
|
||||
self.__set_result(success=True,
|
||||
fileitem=fileitem,
|
||||
target_item=new_item,
|
||||
target_diritem=target_diritem,
|
||||
need_scrape=need_scrape,
|
||||
transfer_type=transfer_type,
|
||||
need_notify=need_notify)
|
||||
return self.result
|
||||
return self.result.copy()
|
||||
finally:
|
||||
self.result = None
|
||||
|
||||
@staticmethod
|
||||
def __transfer_command(fileitem: FileItem, target_storage: str,
|
||||
@@ -304,7 +330,7 @@ class TransHandler:
|
||||
"""
|
||||
return FileItem(
|
||||
storage=target_storage,
|
||||
path=str(_path).replace("\\", "/"),
|
||||
path=_path.as_posix(),
|
||||
name=_path.name,
|
||||
basename=_path.stem,
|
||||
type="file",
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import random
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import List, Optional, Tuple, Union
|
||||
|
||||
@@ -5,10 +7,11 @@ from app.core.config import settings
|
||||
from app.core.context import TorrentInfo
|
||||
from app.db.site_oper import SiteOper
|
||||
from app.helper.module import ModuleHelper
|
||||
from app.helper.sites import SitesHelper, SiteSpider
|
||||
from app.helper.sites import SitesHelper
|
||||
from app.log import logger
|
||||
from app.modules import _ModuleBase
|
||||
from app.modules.indexer.parser import SiteParserBase
|
||||
from app.modules.indexer.spider import SiteSpider
|
||||
from app.modules.indexer.spider.haidan import HaiDanSpider
|
||||
from app.modules.indexer.spider.hddolby import HddolbySpider
|
||||
from app.modules.indexer.spider.mtorrent import MTorrentSpider
|
||||
@@ -113,6 +116,7 @@ class IndexerModule(_ModuleBase):
|
||||
|
||||
# 搜索多个关键字
|
||||
error_flag = False
|
||||
search_count = 0
|
||||
for search_word in keywords:
|
||||
# 可能为关键字或ttxxxx
|
||||
if search_word \
|
||||
@@ -128,6 +132,11 @@ class IndexerModule(_ModuleBase):
|
||||
logger.warn(msg)
|
||||
continue
|
||||
|
||||
if search_count > 0:
|
||||
# 强制休眠 1-10 秒
|
||||
logger.info(f"站点 {site.get('name')} 已搜索 {search_count} 次,强制休眠 1-10 秒 ...")
|
||||
time.sleep(random.randint(1, 10))
|
||||
|
||||
# 去除搜索关键字中的特殊字符
|
||||
if search_word:
|
||||
search_word = StringUtils.clear(search_word, replace_word=" ", allow_space=True)
|
||||
@@ -187,6 +196,8 @@ class IndexerModule(_ModuleBase):
|
||||
break
|
||||
except Exception as err:
|
||||
logger.error(f"{site.get('name')} 搜索出错:{str(err)}")
|
||||
finally:
|
||||
search_count += 1
|
||||
|
||||
# 索引花费的时间
|
||||
seconds = (datetime.now() - start_time).seconds
|
||||
@@ -200,10 +211,10 @@ class IndexerModule(_ModuleBase):
|
||||
|
||||
# 返回结果
|
||||
if not result_array or len(result_array) == 0:
|
||||
logger.warn(f"{site.get('name')} 未搜索到数据,耗时 {seconds} 秒")
|
||||
logger.warn(f"{site.get('name')} 未搜索到数据,共搜索 {search_count} 次,耗时 {seconds} 秒")
|
||||
return []
|
||||
else:
|
||||
logger.info(f"{site.get('name')} 搜索完成,耗时 {seconds} 秒,返回数据:{len(result_array)}")
|
||||
logger.info(f"{site.get('name')} 搜索完成,共搜索 {search_count} 次,耗时 {seconds} 秒,返回数据:{len(result_array)}")
|
||||
# TorrentInfo
|
||||
torrents = [TorrentInfo(site=site.get("id"),
|
||||
site_name=site.get("name"),
|
||||
|
||||
@@ -14,15 +14,18 @@ class DiscuzUserInfo(SiteParserBase):
|
||||
def _parse_user_base_info(self, html_text: str):
|
||||
html_text = self._prepare_html_text(html_text)
|
||||
html = etree.HTML(html_text)
|
||||
|
||||
user_info = html.xpath('//a[contains(@href, "&uid=")]')
|
||||
if user_info:
|
||||
user_id_match = re.search(r"&uid=(\d+)", user_info[0].attrib['href'])
|
||||
if user_id_match and user_id_match.group().strip():
|
||||
self.userid = user_id_match.group(1)
|
||||
self._torrent_seeding_page = f"forum.php?&mod=torrents&cat_5up=on"
|
||||
self._user_detail_page = user_info[0].attrib['href']
|
||||
self.username = user_info[0].text.strip()
|
||||
try:
|
||||
user_info = html.xpath('//a[contains(@href, "&uid=")]')
|
||||
if user_info:
|
||||
user_id_match = re.search(r"&uid=(\d+)", user_info[0].attrib['href'])
|
||||
if user_id_match and user_id_match.group().strip():
|
||||
self.userid = user_id_match.group(1)
|
||||
self._torrent_seeding_page = f"forum.php?&mod=torrents&cat_5up=on"
|
||||
self._user_detail_page = user_info[0].attrib['href']
|
||||
self.username = user_info[0].text.strip()
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_site_page(self, html_text: str):
|
||||
pass
|
||||
@@ -34,40 +37,44 @@ class DiscuzUserInfo(SiteParserBase):
|
||||
:return:
|
||||
"""
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
|
||||
# 用户等级
|
||||
user_levels_text = html.xpath('//a[contains(@href, "usergroup")]/text()')
|
||||
if user_levels_text:
|
||||
self.user_level = user_levels_text[-1].strip()
|
||||
# 用户等级
|
||||
user_levels_text = html.xpath('//a[contains(@href, "usergroup")]/text()')
|
||||
if user_levels_text:
|
||||
self.user_level = user_levels_text[-1].strip()
|
||||
|
||||
# 加入日期
|
||||
join_at_text = html.xpath('//li[em[text()="注册时间"]]/text()')
|
||||
if join_at_text:
|
||||
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].strip())
|
||||
# 加入日期
|
||||
join_at_text = html.xpath('//li[em[text()="注册时间"]]/text()')
|
||||
if join_at_text:
|
||||
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].strip())
|
||||
|
||||
# 分享率
|
||||
ratio_text = html.xpath('//li[contains(.//text(), "分享率")]//text()')
|
||||
if ratio_text:
|
||||
ratio_match = re.search(r"\(([\d,.]+)\)", ratio_text[0])
|
||||
if ratio_match and ratio_match.group(1).strip():
|
||||
self.bonus = StringUtils.str_float(ratio_match.group(1))
|
||||
# 分享率
|
||||
ratio_text = html.xpath('//li[contains(.//text(), "分享率")]//text()')
|
||||
if ratio_text:
|
||||
ratio_match = re.search(r"\(([\d,.]+)\)", ratio_text[0])
|
||||
if ratio_match and ratio_match.group(1).strip():
|
||||
self.bonus = StringUtils.str_float(ratio_match.group(1))
|
||||
|
||||
# 积分
|
||||
bouns_text = html.xpath('//li[em[text()="积分"]]/text()')
|
||||
if bouns_text:
|
||||
self.bonus = StringUtils.str_float(bouns_text[0].strip())
|
||||
# 积分
|
||||
bouns_text = html.xpath('//li[em[text()="积分"]]/text()')
|
||||
if bouns_text:
|
||||
self.bonus = StringUtils.str_float(bouns_text[0].strip())
|
||||
|
||||
# 上传
|
||||
upload_text = html.xpath('//li[em[contains(text(),"上传量")]]/text()')
|
||||
if upload_text:
|
||||
self.upload = StringUtils.num_filesize(upload_text[0].strip().split('/')[-1])
|
||||
# 上传
|
||||
upload_text = html.xpath('//li[em[contains(text(),"上传量")]]/text()')
|
||||
if upload_text:
|
||||
self.upload = StringUtils.num_filesize(upload_text[0].strip().split('/')[-1])
|
||||
|
||||
# 下载
|
||||
download_text = html.xpath('//li[em[contains(text(),"下载量")]]/text()')
|
||||
if download_text:
|
||||
self.download = StringUtils.num_filesize(download_text[0].strip().split('/')[-1])
|
||||
# 下载
|
||||
download_text = html.xpath('//li[em[contains(text(),"下载量")]]/text()')
|
||||
if download_text:
|
||||
self.download = StringUtils.num_filesize(download_text[0].strip().split('/')[-1])
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: bool = False) -> Optional[str]:
|
||||
"""
|
||||
@@ -77,44 +84,48 @@ class DiscuzUserInfo(SiteParserBase):
|
||||
:return: 下页地址
|
||||
"""
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
|
||||
size_col = 3
|
||||
seeders_col = 4
|
||||
# 搜索size列
|
||||
if html.xpath('//tr[position()=1]/td[.//img[@class="size"] and .//img[@alt="size"]]'):
|
||||
size_col = len(html.xpath('//tr[position()=1]/td[.//img[@class="size"] '
|
||||
'and .//img[@alt="size"]]/preceding-sibling::td')) + 1
|
||||
# 搜索seeders列
|
||||
if html.xpath('//tr[position()=1]/td[.//img[@class="seeders"] and .//img[@alt="seeders"]]'):
|
||||
seeders_col = len(html.xpath('//tr[position()=1]/td[.//img[@class="seeders"] '
|
||||
'and .//img[@alt="seeders"]]/preceding-sibling::td')) + 1
|
||||
size_col = 3
|
||||
seeders_col = 4
|
||||
# 搜索size列
|
||||
if html.xpath('//tr[position()=1]/td[.//img[@class="size"] and .//img[@alt="size"]]'):
|
||||
size_col = len(html.xpath('//tr[position()=1]/td[.//img[@class="size"] '
|
||||
'and .//img[@alt="size"]]/preceding-sibling::td')) + 1
|
||||
# 搜索seeders列
|
||||
if html.xpath('//tr[position()=1]/td[.//img[@class="seeders"] and .//img[@alt="seeders"]]'):
|
||||
seeders_col = len(html.xpath('//tr[position()=1]/td[.//img[@class="seeders"] '
|
||||
'and .//img[@alt="seeders"]]/preceding-sibling::td')) + 1
|
||||
|
||||
page_seeding = 0
|
||||
page_seeding_size = 0
|
||||
page_seeding_info = []
|
||||
seeding_sizes = html.xpath(f'//tr[position()>1]/td[{size_col}]')
|
||||
seeding_seeders = html.xpath(f'//tr[position()>1]/td[{seeders_col}]//text()')
|
||||
if seeding_sizes and seeding_seeders:
|
||||
page_seeding = len(seeding_sizes)
|
||||
page_seeding = 0
|
||||
page_seeding_size = 0
|
||||
page_seeding_info = []
|
||||
seeding_sizes = html.xpath(f'//tr[position()>1]/td[{size_col}]')
|
||||
seeding_seeders = html.xpath(f'//tr[position()>1]/td[{seeders_col}]//text()')
|
||||
if seeding_sizes and seeding_seeders:
|
||||
page_seeding = len(seeding_sizes)
|
||||
|
||||
for i in range(0, len(seeding_sizes)):
|
||||
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
|
||||
seeders = StringUtils.str_int(seeding_seeders[i])
|
||||
for i in range(0, len(seeding_sizes)):
|
||||
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
|
||||
seeders = StringUtils.str_int(seeding_seeders[i])
|
||||
|
||||
page_seeding_size += size
|
||||
page_seeding_info.append([seeders, size])
|
||||
page_seeding_size += size
|
||||
page_seeding_info.append([seeders, size])
|
||||
|
||||
self.seeding += page_seeding
|
||||
self.seeding_size += page_seeding_size
|
||||
self.seeding_info.extend(page_seeding_info)
|
||||
self.seeding += page_seeding
|
||||
self.seeding_size += page_seeding_size
|
||||
self.seeding_info.extend(page_seeding_info)
|
||||
|
||||
# 是否存在下页数据
|
||||
next_page = None
|
||||
next_page_text = html.xpath('//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁")]/@href')
|
||||
if next_page_text:
|
||||
next_page = next_page_text[-1].strip()
|
||||
# 是否存在下页数据
|
||||
next_page = None
|
||||
next_page_text = html.xpath('//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁")]/@href')
|
||||
if next_page_text:
|
||||
next_page = next_page_text[-1].strip()
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
return next_page
|
||||
|
||||
|
||||
@@ -24,10 +24,13 @@ class FileListSiteUserInfo(SiteParserBase):
|
||||
def _parse_user_base_info(self, html_text: str):
|
||||
html_text = self._prepare_html_text(html_text)
|
||||
html = etree.HTML(html_text)
|
||||
|
||||
ret = html.xpath(f'//a[contains(@href, "userdetails") and contains(@href, "{self.userid}")]//text()')
|
||||
if ret:
|
||||
self.username = str(ret[0])
|
||||
try:
|
||||
ret = html.xpath(f'//a[contains(@href, "userdetails") and contains(@href, "{self.userid}")]//text()')
|
||||
if ret:
|
||||
self.username = str(ret[0])
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_user_traffic_info(self, html_text: str):
|
||||
"""
|
||||
@@ -40,39 +43,41 @@ class FileListSiteUserInfo(SiteParserBase):
|
||||
def _parse_user_detail_info(self, html_text: str):
|
||||
html_text = self._prepare_html_text(html_text)
|
||||
html = etree.HTML(html_text)
|
||||
try:
|
||||
upload_html = html.xpath('//table//tr/td[text()="Uploaded"]/following-sibling::td//text()')
|
||||
if upload_html:
|
||||
self.upload = StringUtils.num_filesize(upload_html[0])
|
||||
download_html = html.xpath('//table//tr/td[text()="Downloaded"]/following-sibling::td//text()')
|
||||
if download_html:
|
||||
self.download = StringUtils.num_filesize(download_html[0])
|
||||
|
||||
upload_html = html.xpath('//table//tr/td[text()="Uploaded"]/following-sibling::td//text()')
|
||||
if upload_html:
|
||||
self.upload = StringUtils.num_filesize(upload_html[0])
|
||||
download_html = html.xpath('//table//tr/td[text()="Downloaded"]/following-sibling::td//text()')
|
||||
if download_html:
|
||||
self.download = StringUtils.num_filesize(download_html[0])
|
||||
ratio_html = html.xpath('//table//tr/td[text()="Share ratio"]/following-sibling::td//text()')
|
||||
if ratio_html:
|
||||
share_ratio = StringUtils.str_float(ratio_html[0])
|
||||
else:
|
||||
share_ratio = 0
|
||||
self.ratio = 0 if self.download == 0 else share_ratio
|
||||
|
||||
ratio_html = html.xpath('//table//tr/td[text()="Share ratio"]/following-sibling::td//text()')
|
||||
if ratio_html:
|
||||
share_ratio = StringUtils.str_float(ratio_html[0])
|
||||
else:
|
||||
share_ratio = 0
|
||||
self.ratio = 0 if self.download == 0 else share_ratio
|
||||
seed_html = html.xpath('//table//tr/td[text()="Seed bonus"]/following-sibling::td//text()')
|
||||
if seed_html:
|
||||
self.seeding = StringUtils.str_int(seed_html[1])
|
||||
self.seeding_size = StringUtils.num_filesize(seed_html[3])
|
||||
|
||||
seed_html = html.xpath('//table//tr/td[text()="Seed bonus"]/following-sibling::td//text()')
|
||||
if seed_html:
|
||||
self.seeding = StringUtils.str_int(seed_html[1])
|
||||
self.seeding_size = StringUtils.num_filesize(seed_html[3])
|
||||
user_level_html = html.xpath('//table//tr/td[text()="Class"]/following-sibling::td//text()')
|
||||
if user_level_html:
|
||||
self.user_level = user_level_html[0].strip()
|
||||
|
||||
user_level_html = html.xpath('//table//tr/td[text()="Class"]/following-sibling::td//text()')
|
||||
if user_level_html:
|
||||
self.user_level = user_level_html[0].strip()
|
||||
join_at_html = html.xpath('//table//tr/td[contains(text(), "Join")]/following-sibling::td//text()')
|
||||
if join_at_html:
|
||||
join_at = (join_at_html[0].split("("))[0].strip()
|
||||
self.join_at = StringUtils.unify_datetime_str(join_at)
|
||||
|
||||
join_at_html = html.xpath('//table//tr/td[contains(text(), "Join")]/following-sibling::td//text()')
|
||||
if join_at_html:
|
||||
join_at = (join_at_html[0].split("("))[0].strip()
|
||||
self.join_at = StringUtils.unify_datetime_str(join_at)
|
||||
|
||||
bonus_html = html.xpath('//a[contains(@href, "shop.php")]')
|
||||
if bonus_html:
|
||||
self.bonus = StringUtils.str_float(bonus_html[0].xpath("string(.)").strip())
|
||||
pass
|
||||
bonus_html = html.xpath('//a[contains(@href, "shop.php")]')
|
||||
if bonus_html:
|
||||
self.bonus = StringUtils.str_float(bonus_html[0].xpath("string(.)").strip())
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: Optional[bool] = False) -> Optional[str]:
|
||||
"""
|
||||
@@ -82,28 +87,32 @@ class FileListSiteUserInfo(SiteParserBase):
|
||||
:return: 下页地址
|
||||
"""
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
|
||||
size_col = 6
|
||||
seeders_col = 7
|
||||
size_col = 6
|
||||
seeders_col = 7
|
||||
|
||||
page_seeding_size = 0
|
||||
page_seeding_info = []
|
||||
seeding_sizes = html.xpath(f'//table/tr[position()>1]/td[{size_col}]')
|
||||
seeding_seeders = html.xpath(f'//table/tr[position()>1]/td[{seeders_col}]')
|
||||
if seeding_sizes and seeding_seeders:
|
||||
for i in range(0, len(seeding_sizes)):
|
||||
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
|
||||
seeders = StringUtils.str_int(seeding_seeders[i].xpath("string(.)").strip())
|
||||
page_seeding_size = 0
|
||||
page_seeding_info = []
|
||||
seeding_sizes = html.xpath(f'//table/tr[position()>1]/td[{size_col}]')
|
||||
seeding_seeders = html.xpath(f'//table/tr[position()>1]/td[{seeders_col}]')
|
||||
if seeding_sizes and seeding_seeders:
|
||||
for i in range(0, len(seeding_sizes)):
|
||||
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
|
||||
seeders = StringUtils.str_int(seeding_seeders[i].xpath("string(.)").strip())
|
||||
|
||||
page_seeding_size += size
|
||||
page_seeding_info.append([seeders, size])
|
||||
page_seeding_size += size
|
||||
page_seeding_info.append([seeders, size])
|
||||
|
||||
self.seeding_info.extend(page_seeding_info)
|
||||
self.seeding_info.extend(page_seeding_info)
|
||||
|
||||
# 是否存在下页数据
|
||||
next_page = None
|
||||
# 是否存在下页数据
|
||||
next_page = None
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
return next_page
|
||||
|
||||
|
||||
@@ -14,46 +14,49 @@ class GazelleSiteUserInfo(SiteParserBase):
|
||||
def _parse_user_base_info(self, html_text: str):
|
||||
html_text = self._prepare_html_text(html_text)
|
||||
html = etree.HTML(html_text)
|
||||
try:
|
||||
tmps = html.xpath('//a[contains(@href, "user.php?id=")]')
|
||||
if tmps:
|
||||
user_id_match = re.search(r"user.php\?id=(\d+)", tmps[0].attrib['href'])
|
||||
if user_id_match and user_id_match.group().strip():
|
||||
self.userid = user_id_match.group(1)
|
||||
self._torrent_seeding_page = f"torrents.php?type=seeding&userid={self.userid}"
|
||||
self._user_detail_page = f"user.php?id={self.userid}"
|
||||
self.username = tmps[0].text.strip()
|
||||
|
||||
tmps = html.xpath('//a[contains(@href, "user.php?id=")]')
|
||||
if tmps:
|
||||
user_id_match = re.search(r"user.php\?id=(\d+)", tmps[0].attrib['href'])
|
||||
if user_id_match and user_id_match.group().strip():
|
||||
self.userid = user_id_match.group(1)
|
||||
self._torrent_seeding_page = f"torrents.php?type=seeding&userid={self.userid}"
|
||||
self._user_detail_page = f"user.php?id={self.userid}"
|
||||
self.username = tmps[0].text.strip()
|
||||
|
||||
tmps = html.xpath('//*[@id="header-uploaded-value"]/@data-value')
|
||||
if tmps:
|
||||
self.upload = StringUtils.num_filesize(tmps[0])
|
||||
else:
|
||||
tmps = html.xpath('//li[@id="stats_seeding"]/span/text()')
|
||||
tmps = html.xpath('//*[@id="header-uploaded-value"]/@data-value')
|
||||
if tmps:
|
||||
self.upload = StringUtils.num_filesize(tmps[0])
|
||||
else:
|
||||
tmps = html.xpath('//li[@id="stats_seeding"]/span/text()')
|
||||
if tmps:
|
||||
self.upload = StringUtils.num_filesize(tmps[0])
|
||||
|
||||
tmps = html.xpath('//*[@id="header-downloaded-value"]/@data-value')
|
||||
if tmps:
|
||||
self.download = StringUtils.num_filesize(tmps[0])
|
||||
else:
|
||||
tmps = html.xpath('//li[@id="stats_leeching"]/span/text()')
|
||||
tmps = html.xpath('//*[@id="header-downloaded-value"]/@data-value')
|
||||
if tmps:
|
||||
self.download = StringUtils.num_filesize(tmps[0])
|
||||
else:
|
||||
tmps = html.xpath('//li[@id="stats_leeching"]/span/text()')
|
||||
if tmps:
|
||||
self.download = StringUtils.num_filesize(tmps[0])
|
||||
|
||||
self.ratio = 0.0 if self.download <= 0.0 else round(self.upload / self.download, 3)
|
||||
self.ratio = 0.0 if self.download <= 0.0 else round(self.upload / self.download, 3)
|
||||
|
||||
tmps = html.xpath('//a[contains(@href, "bonus.php")]/@data-tooltip')
|
||||
if tmps:
|
||||
bonus_match = re.search(r"([\d,.]+)", tmps[0])
|
||||
if bonus_match and bonus_match.group(1).strip():
|
||||
self.bonus = StringUtils.str_float(bonus_match.group(1))
|
||||
else:
|
||||
tmps = html.xpath('//a[contains(@href, "bonus.php")]')
|
||||
tmps = html.xpath('//a[contains(@href, "bonus.php")]/@data-tooltip')
|
||||
if tmps:
|
||||
bonus_text = tmps[0].xpath("string(.)")
|
||||
bonus_match = re.search(r"([\d,.]+)", bonus_text)
|
||||
bonus_match = re.search(r"([\d,.]+)", tmps[0])
|
||||
if bonus_match and bonus_match.group(1).strip():
|
||||
self.bonus = StringUtils.str_float(bonus_match.group(1))
|
||||
else:
|
||||
tmps = html.xpath('//a[contains(@href, "bonus.php")]')
|
||||
if tmps:
|
||||
bonus_text = tmps[0].xpath("string(.)")
|
||||
bonus_match = re.search(r"([\d,.]+)", bonus_text)
|
||||
if bonus_match and bonus_match.group(1).strip():
|
||||
self.bonus = StringUtils.str_float(bonus_match.group(1))
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_site_page(self, html_text: str):
|
||||
pass
|
||||
@@ -65,27 +68,31 @@ class GazelleSiteUserInfo(SiteParserBase):
|
||||
:return:
|
||||
"""
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
|
||||
# 用户等级
|
||||
user_levels_text = html.xpath('//*[@id="class-value"]/@data-value')
|
||||
if user_levels_text:
|
||||
self.user_level = user_levels_text[0].strip()
|
||||
else:
|
||||
user_levels_text = html.xpath('//li[contains(text(), "用户等级")]/text()')
|
||||
# 用户等级
|
||||
user_levels_text = html.xpath('//*[@id="class-value"]/@data-value')
|
||||
if user_levels_text:
|
||||
self.user_level = user_levels_text[0].split(':')[1].strip()
|
||||
self.user_level = user_levels_text[0].strip()
|
||||
else:
|
||||
user_levels_text = html.xpath('//li[contains(text(), "用户等级")]/text()')
|
||||
if user_levels_text:
|
||||
self.user_level = user_levels_text[0].split(':')[1].strip()
|
||||
|
||||
# 加入日期
|
||||
join_at_text = html.xpath('//*[@id="join-date-value"]/@data-value')
|
||||
if join_at_text:
|
||||
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].strip())
|
||||
else:
|
||||
join_at_text = html.xpath(
|
||||
'//div[contains(@class, "box_userinfo_stats")]//li[contains(text(), "加入时间")]/span/text()')
|
||||
# 加入日期
|
||||
join_at_text = html.xpath('//*[@id="join-date-value"]/@data-value')
|
||||
if join_at_text:
|
||||
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].strip())
|
||||
else:
|
||||
join_at_text = html.xpath(
|
||||
'//div[contains(@class, "box_userinfo_stats")]//li[contains(text(), "加入时间")]/span/text()')
|
||||
if join_at_text:
|
||||
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].strip())
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: Optional[bool] = False) -> Optional[str]:
|
||||
"""
|
||||
@@ -95,48 +102,52 @@ class GazelleSiteUserInfo(SiteParserBase):
|
||||
:return: 下页地址
|
||||
"""
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
|
||||
size_col = 3
|
||||
# 搜索size列
|
||||
if html.xpath('//table[contains(@id, "torrent")]//tr[1]/td'):
|
||||
size_col = len(html.xpath('//table[contains(@id, "torrent")]//tr[1]/td')) - 3
|
||||
# 搜索seeders列
|
||||
seeders_col = size_col + 2
|
||||
size_col = 3
|
||||
# 搜索size列
|
||||
if html.xpath('//table[contains(@id, "torrent")]//tr[1]/td'):
|
||||
size_col = len(html.xpath('//table[contains(@id, "torrent")]//tr[1]/td')) - 3
|
||||
# 搜索seeders列
|
||||
seeders_col = size_col + 2
|
||||
|
||||
page_seeding = 0
|
||||
page_seeding_size = 0
|
||||
page_seeding_info = []
|
||||
seeding_sizes = html.xpath(f'//table[contains(@id, "torrent")]//tr[position()>1]/td[{size_col}]')
|
||||
seeding_seeders = html.xpath(f'//table[contains(@id, "torrent")]//tr[position()>1]/td[{seeders_col}]/text()')
|
||||
if seeding_sizes and seeding_seeders:
|
||||
page_seeding = len(seeding_sizes)
|
||||
page_seeding = 0
|
||||
page_seeding_size = 0
|
||||
page_seeding_info = []
|
||||
seeding_sizes = html.xpath(f'//table[contains(@id, "torrent")]//tr[position()>1]/td[{size_col}]')
|
||||
seeding_seeders = html.xpath(f'//table[contains(@id, "torrent")]//tr[position()>1]/td[{seeders_col}]/text()')
|
||||
if seeding_sizes and seeding_seeders:
|
||||
page_seeding = len(seeding_sizes)
|
||||
|
||||
for i in range(0, len(seeding_sizes)):
|
||||
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
|
||||
seeders = int(seeding_seeders[i])
|
||||
for i in range(0, len(seeding_sizes)):
|
||||
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
|
||||
seeders = int(seeding_seeders[i])
|
||||
|
||||
page_seeding_size += size
|
||||
page_seeding_info.append([seeders, size])
|
||||
page_seeding_size += size
|
||||
page_seeding_info.append([seeders, size])
|
||||
|
||||
if multi_page:
|
||||
self.seeding += page_seeding
|
||||
self.seeding_size += page_seeding_size
|
||||
self.seeding_info.extend(page_seeding_info)
|
||||
else:
|
||||
if not self.seeding:
|
||||
self.seeding = page_seeding
|
||||
if not self.seeding_size:
|
||||
self.seeding_size = page_seeding_size
|
||||
if not self.seeding_info:
|
||||
self.seeding_info = page_seeding_info
|
||||
if multi_page:
|
||||
self.seeding += page_seeding
|
||||
self.seeding_size += page_seeding_size
|
||||
self.seeding_info.extend(page_seeding_info)
|
||||
else:
|
||||
if not self.seeding:
|
||||
self.seeding = page_seeding
|
||||
if not self.seeding_size:
|
||||
self.seeding_size = page_seeding_size
|
||||
if not self.seeding_info:
|
||||
self.seeding_info = page_seeding_info
|
||||
|
||||
# 是否存在下页数据
|
||||
next_page = None
|
||||
next_page_text = html.xpath('//a[contains(.//text(), "Next") or contains(.//text(), "下一页")]/@href')
|
||||
if next_page_text:
|
||||
next_page = next_page_text[-1].strip()
|
||||
# 是否存在下页数据
|
||||
next_page = None
|
||||
next_page_text = html.xpath('//a[contains(.//text(), "Next") or contains(.//text(), "下一页")]/@href')
|
||||
if next_page_text:
|
||||
next_page = next_page_text[-1].strip()
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
return next_page
|
||||
|
||||
|
||||
@@ -14,67 +14,79 @@ class IptSiteUserInfo(SiteParserBase):
|
||||
def _parse_user_base_info(self, html_text: str):
|
||||
html_text = self._prepare_html_text(html_text)
|
||||
html = etree.HTML(html_text)
|
||||
tmps = html.xpath('//a[contains(@href, "/u/")]//text()')
|
||||
tmps_id = html.xpath('//a[contains(@href, "/u/")]/@href')
|
||||
if tmps:
|
||||
self.username = str(tmps[-1])
|
||||
if tmps_id:
|
||||
user_id_match = re.search(r"/u/(\d+)", tmps_id[0])
|
||||
if user_id_match and user_id_match.group().strip():
|
||||
self.userid = user_id_match.group(1)
|
||||
self._user_detail_page = f"user.php?u={self.userid}"
|
||||
self._torrent_seeding_page = f"peers?u={self.userid}"
|
||||
try:
|
||||
tmps = html.xpath('//a[contains(@href, "/u/")]//text()')
|
||||
tmps_id = html.xpath('//a[contains(@href, "/u/")]/@href')
|
||||
if tmps:
|
||||
self.username = str(tmps[-1])
|
||||
if tmps_id:
|
||||
user_id_match = re.search(r"/u/(\d+)", tmps_id[0])
|
||||
if user_id_match and user_id_match.group().strip():
|
||||
self.userid = user_id_match.group(1)
|
||||
self._user_detail_page = f"user.php?u={self.userid}"
|
||||
self._torrent_seeding_page = f"peers?u={self.userid}"
|
||||
|
||||
tmps = html.xpath('//div[@class = "stats"]/div/div')
|
||||
if tmps:
|
||||
self.upload = StringUtils.num_filesize(str(tmps[0].xpath('span/text()')[1]).strip())
|
||||
self.download = StringUtils.num_filesize(str(tmps[0].xpath('span/text()')[2]).strip())
|
||||
self.seeding = StringUtils.str_int(tmps[0].xpath('a')[2].xpath('text()')[0])
|
||||
self.leeching = StringUtils.str_int(tmps[0].xpath('a')[2].xpath('text()')[1])
|
||||
self.ratio = StringUtils.str_float(str(tmps[0].xpath('span/text()')[0]).strip().replace('-', '0'))
|
||||
self.bonus = StringUtils.str_float(tmps[0].xpath('a')[3].xpath('text()')[0])
|
||||
tmps = html.xpath('//div[@class = "stats"]/div/div')
|
||||
if tmps:
|
||||
self.upload = StringUtils.num_filesize(str(tmps[0].xpath('span/text()')[1]).strip())
|
||||
self.download = StringUtils.num_filesize(str(tmps[0].xpath('span/text()')[2]).strip())
|
||||
self.seeding = StringUtils.str_int(tmps[0].xpath('a')[2].xpath('text()')[0])
|
||||
self.leeching = StringUtils.str_int(tmps[0].xpath('a')[2].xpath('text()')[1])
|
||||
self.ratio = StringUtils.str_float(str(tmps[0].xpath('span/text()')[0]).strip().replace('-', '0'))
|
||||
self.bonus = StringUtils.str_float(tmps[0].xpath('a')[3].xpath('text()')[0])
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_site_page(self, html_text: str):
|
||||
pass
|
||||
|
||||
def _parse_user_detail_info(self, html_text: str):
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return
|
||||
|
||||
user_levels_text = html.xpath('//tr/th[text()="Class"]/following-sibling::td[1]/text()')
|
||||
if user_levels_text:
|
||||
self.user_level = user_levels_text[0].strip()
|
||||
user_levels_text = html.xpath('//tr/th[text()="Class"]/following-sibling::td[1]/text()')
|
||||
if user_levels_text:
|
||||
self.user_level = user_levels_text[0].strip()
|
||||
|
||||
# 加入日期
|
||||
join_at_text = html.xpath('//tr/th[text()="Join date"]/following-sibling::td[1]/text()')
|
||||
if join_at_text:
|
||||
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].split(' (')[0])
|
||||
# 加入日期
|
||||
join_at_text = html.xpath('//tr/th[text()="Join date"]/following-sibling::td[1]/text()')
|
||||
if join_at_text:
|
||||
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].split(' (')[0])
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: bool = False) -> Optional[str]:
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
# seeding start
|
||||
seeding_end_pos = 3
|
||||
if html.xpath('//tr/td[text() = "Leechers"]'):
|
||||
seeding_end_pos = len(html.xpath('//tr/td[text() = "Leechers"]/../preceding-sibling::tr')) + 1
|
||||
seeding_end_pos = seeding_end_pos - 3
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
# seeding start
|
||||
seeding_end_pos = 3
|
||||
if html.xpath('//tr/td[text() = "Leechers"]'):
|
||||
seeding_end_pos = len(html.xpath('//tr/td[text() = "Leechers"]/../preceding-sibling::tr')) + 1
|
||||
seeding_end_pos = seeding_end_pos - 3
|
||||
|
||||
page_seeding = 0
|
||||
page_seeding_size = 0
|
||||
seeding_torrents = html.xpath('//tr/td[text() = "Seeders"]/../following-sibling::tr/td[position()=6]/text()')
|
||||
if seeding_torrents:
|
||||
page_seeding = seeding_end_pos
|
||||
for per_size in seeding_torrents[:seeding_end_pos]:
|
||||
if '(' in per_size and ')' in per_size:
|
||||
per_size = per_size.split('(')[-1]
|
||||
per_size = per_size.split(')')[0]
|
||||
page_seeding = 0
|
||||
page_seeding_size = 0
|
||||
seeding_torrents = html.xpath('//tr/td[text() = "Seeders"]/../following-sibling::tr/td[position()=6]/text()')
|
||||
if seeding_torrents:
|
||||
page_seeding = seeding_end_pos
|
||||
for per_size in seeding_torrents[:seeding_end_pos]:
|
||||
if '(' in per_size and ')' in per_size:
|
||||
per_size = per_size.split('(')[-1]
|
||||
per_size = per_size.split(')')[0]
|
||||
|
||||
page_seeding_size += StringUtils.num_filesize(per_size)
|
||||
page_seeding_size += StringUtils.num_filesize(per_size)
|
||||
|
||||
self.seeding = page_seeding
|
||||
self.seeding_size = page_seeding_size
|
||||
self.seeding = page_seeding
|
||||
self.seeding_size = page_seeding_size
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_user_traffic_info(self, html_text: str):
|
||||
pass
|
||||
|
||||
@@ -23,12 +23,16 @@ class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
|
||||
if not html_text:
|
||||
return
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return
|
||||
total_row = html.xpath('//table[@class="table table-bordered"]//tr[td[1][normalize-space()="Total"]]')
|
||||
if not total_row:
|
||||
return
|
||||
seeding_count = total_row[0].xpath('./td[2]/text()')
|
||||
seeding_size = total_row[0].xpath('./td[3]/text()')
|
||||
self.seeding = StringUtils.str_int(seeding_count[0]) if seeding_count else 0
|
||||
self.seeding_size = StringUtils.num_filesize(seeding_size[0].strip()) if seeding_size else 0
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return
|
||||
total_row = html.xpath('//table[@class="table table-bordered"]//tr[td[1][normalize-space()="Total"]]')
|
||||
if not total_row:
|
||||
return
|
||||
seeding_count = total_row[0].xpath('./td[2]/text()')
|
||||
seeding_size = total_row[0].xpath('./td[3]/text()')
|
||||
self.seeding = StringUtils.str_int(seeding_count[0]) if seeding_count else 0
|
||||
self.seeding_size = StringUtils.num_filesize(seeding_size[0].strip()) if seeding_size else 0
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
@@ -17,21 +17,25 @@ class NexusHhanclubSiteUserInfo(NexusPhpSiteUserInfo):
|
||||
html_text = self._prepare_html_text(html_text)
|
||||
html = etree.HTML(html_text)
|
||||
|
||||
# 上传、下载、分享率
|
||||
upload_match = re.search(r"[_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)",
|
||||
html.xpath('//*[@id="user-info-panel"]/div[2]/div[2]/div[4]/text()')[0])
|
||||
download_match = re.search(r"[_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)",
|
||||
html.xpath('//*[@id="user-info-panel"]/div[2]/div[2]/div[5]/text()')[0])
|
||||
ratio_match = re.search(r"分享率][::_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+)",
|
||||
html.xpath('//*[@id="user-info-panel"]/div[2]/div[1]/div[1]/div/text()')[0])
|
||||
try:
|
||||
# 上传、下载、分享率
|
||||
upload_match = re.search(r"[_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)",
|
||||
html.xpath('//*[@id="user-info-panel"]/div[2]/div[2]/div[4]/text()')[0])
|
||||
download_match = re.search(r"[_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)",
|
||||
html.xpath('//*[@id="user-info-panel"]/div[2]/div[2]/div[5]/text()')[0])
|
||||
ratio_match = re.search(r"分享率][::_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+)",
|
||||
html.xpath('//*[@id="user-info-panel"]/div[2]/div[1]/div[1]/div/text()')[0])
|
||||
|
||||
# 计算分享率
|
||||
self.upload = StringUtils.num_filesize(upload_match.group(1).strip()) if upload_match else 0
|
||||
self.download = StringUtils.num_filesize(download_match.group(1).strip()) if download_match else 0
|
||||
# 优先使用页面上的分享率
|
||||
calc_ratio = 0.0 if self.download <= 0.0 else round(self.upload / self.download, 3)
|
||||
self.ratio = StringUtils.str_float(ratio_match.group(1)) if (
|
||||
ratio_match and ratio_match.group(1).strip()) else calc_ratio
|
||||
# 计算分享率
|
||||
self.upload = StringUtils.num_filesize(upload_match.group(1).strip()) if upload_match else 0
|
||||
self.download = StringUtils.num_filesize(download_match.group(1).strip()) if download_match else 0
|
||||
# 优先使用页面上的分享率
|
||||
calc_ratio = 0.0 if self.download <= 0.0 else round(self.upload / self.download, 3)
|
||||
self.ratio = StringUtils.str_float(ratio_match.group(1)) if (
|
||||
ratio_match and ratio_match.group(1).strip()) else calc_ratio
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_user_detail_info(self, html_text: str):
|
||||
"""
|
||||
@@ -42,12 +46,16 @@ class NexusHhanclubSiteUserInfo(NexusPhpSiteUserInfo):
|
||||
super()._parse_user_detail_info(html_text)
|
||||
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return
|
||||
# 加入时间
|
||||
join_at_text = html.xpath('//*[@id="mainContent"]/div/div[2]/div[4]/div[3]/span[2]/text()[1]')
|
||||
if join_at_text:
|
||||
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].split(' (')[0].strip())
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return
|
||||
# 加入时间
|
||||
join_at_text = html.xpath('//*[@id="mainContent"]/div/div[2]/div[4]/div[3]/span[2]/text()[1]')
|
||||
if join_at_text:
|
||||
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].split(' (')[0].strip())
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _get_user_level(self, html):
|
||||
super()._get_user_level(html)
|
||||
|
||||
@@ -34,21 +34,25 @@ class NexusPhpSiteUserInfo(SiteParserBase):
|
||||
:return:
|
||||
"""
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return
|
||||
|
||||
message_labels = html.xpath('//a[@href="messages.php"]/..')
|
||||
message_labels.extend(html.xpath('//a[contains(@href, "messages.php")]/..'))
|
||||
if message_labels:
|
||||
message_text = message_labels[0].xpath("string(.)")
|
||||
message_labels = html.xpath('//a[@href="messages.php"]/..')
|
||||
message_labels.extend(html.xpath('//a[contains(@href, "messages.php")]/..'))
|
||||
if message_labels:
|
||||
message_text = message_labels[0].xpath("string(.)")
|
||||
|
||||
logger.debug(f"{self._site_name} 消息原始信息 {message_text}")
|
||||
message_unread_match = re.findall(r"[^Date](信息箱\s*|\((?![^)]*:)|你有\xa0)(\d+)", message_text)
|
||||
logger.debug(f"{self._site_name} 消息原始信息 {message_text}")
|
||||
message_unread_match = re.findall(r"[^Date](信息箱\s*|\((?![^)]*:)|你有\xa0)(\d+)", message_text)
|
||||
|
||||
if message_unread_match and len(message_unread_match[-1]) == 2:
|
||||
self.message_unread = StringUtils.str_int(message_unread_match[-1][1])
|
||||
elif message_text.isdigit():
|
||||
self.message_unread = StringUtils.str_int(message_text)
|
||||
if message_unread_match and len(message_unread_match[-1]) == 2:
|
||||
self.message_unread = StringUtils.str_int(message_unread_match[-1][1])
|
||||
elif message_text.isdigit():
|
||||
self.message_unread = StringUtils.str_int(message_text)
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_user_base_info(self, html_text: str):
|
||||
"""
|
||||
@@ -61,18 +65,23 @@ class NexusPhpSiteUserInfo(SiteParserBase):
|
||||
self._parse_message_unread(html_text)
|
||||
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return
|
||||
|
||||
ret = html.xpath(f'//a[contains(@href, "userdetails") and contains(@href, "{self.userid}")]//b//text()')
|
||||
if ret:
|
||||
self.username = str(ret[0])
|
||||
return
|
||||
ret = html.xpath(f'//a[contains(@href, "userdetails") and contains(@href, "{self.userid}")]//text()')
|
||||
if ret:
|
||||
self.username = str(ret[0])
|
||||
ret = html.xpath(f'//a[contains(@href, "userdetails") and contains(@href, "{self.userid}")]//b//text()')
|
||||
if ret:
|
||||
self.username = str(ret[0])
|
||||
return
|
||||
ret = html.xpath(f'//a[contains(@href, "userdetails") and contains(@href, "{self.userid}")]//text()')
|
||||
if ret:
|
||||
self.username = str(ret[0])
|
||||
|
||||
ret = html.xpath('//a[contains(@href, "userdetails")]//strong//text()')
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
ret = html.xpath('//a[contains(@href, "userdetails")]//strong//text()')
|
||||
if ret:
|
||||
self.username = str(ret[0])
|
||||
return
|
||||
@@ -98,28 +107,32 @@ class NexusPhpSiteUserInfo(SiteParserBase):
|
||||
self.leeching = StringUtils.str_int(leeching_match.group(2)) if leeching_match and leeching_match.group(
|
||||
2).strip() else 0
|
||||
html = etree.HTML(html_text)
|
||||
has_ucoin, self.bonus = self._parse_ucoin(html)
|
||||
if has_ucoin:
|
||||
return
|
||||
tmps = html.xpath('//a[contains(@href,"mybonus")]/text()') if html else None
|
||||
if tmps:
|
||||
bonus_text = str(tmps[0]).strip()
|
||||
bonus_match = re.search(r"([\d,.]+)", bonus_text)
|
||||
if bonus_match and bonus_match.group(1).strip():
|
||||
self.bonus = StringUtils.str_float(bonus_match.group(1))
|
||||
return
|
||||
bonus_match = re.search(r"mybonus.[\[\]::<>/a-zA-Z_\-=\"'\s#;.(使用魔力值豆]+\s*([\d,.]+)[<()&\s]", html_text)
|
||||
try:
|
||||
if bonus_match and bonus_match.group(1).strip():
|
||||
self.bonus = StringUtils.str_float(bonus_match.group(1))
|
||||
has_ucoin, self.bonus = self._parse_ucoin(html)
|
||||
if has_ucoin:
|
||||
return
|
||||
bonus_match = re.search(r"[魔力值|\]][\[\]::<>/a-zA-Z_\-=\"'\s#;]+\s*([\d,.]+|\"[\d,.]+\")[<>()&\s]",
|
||||
html_text,
|
||||
flags=re.S)
|
||||
if bonus_match and bonus_match.group(1).strip():
|
||||
self.bonus = StringUtils.str_float(bonus_match.group(1).strip('"'))
|
||||
except Exception as err:
|
||||
logger.error(f"{self._site_name} 解析魔力值出错, 错误信息: {str(err)}")
|
||||
tmps = html.xpath('//a[contains(@href,"mybonus")]/text()') if html else None
|
||||
if tmps:
|
||||
bonus_text = str(tmps[0]).strip()
|
||||
bonus_match = re.search(r"([\d,.]+)", bonus_text)
|
||||
if bonus_match and bonus_match.group(1).strip():
|
||||
self.bonus = StringUtils.str_float(bonus_match.group(1))
|
||||
return
|
||||
bonus_match = re.search(r"mybonus.[\[\]::<>/a-zA-Z_\-=\"'\s#;.(使用魔力值豆]+\s*([\d,.]+)[<()&\s]", html_text)
|
||||
try:
|
||||
if bonus_match and bonus_match.group(1).strip():
|
||||
self.bonus = StringUtils.str_float(bonus_match.group(1))
|
||||
return
|
||||
bonus_match = re.search(r"[魔力值|\]][\[\]::<>/a-zA-Z_\-=\"'\s#;]+\s*([\d,.]+|\"[\d,.]+\")[<>()&\s]",
|
||||
html_text,
|
||||
flags=re.S)
|
||||
if bonus_match and bonus_match.group(1).strip():
|
||||
self.bonus = StringUtils.str_float(bonus_match.group(1).strip('"'))
|
||||
except Exception as err:
|
||||
logger.error(f"{self._site_name} 解析魔力值出错, 错误信息: {str(err)}")
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
@staticmethod
|
||||
def _parse_ucoin(html):
|
||||
@@ -155,72 +168,76 @@ class NexusPhpSiteUserInfo(SiteParserBase):
|
||||
:return: 下页地址
|
||||
"""
|
||||
html = etree.HTML(str(html_text).replace(r'\/', '/'))
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
|
||||
# 首页存在扩展链接,使用扩展链接
|
||||
seeding_url_text = html.xpath('//a[contains(@href,"torrents.php") '
|
||||
'and contains(@href,"seeding")]/@href')
|
||||
if multi_page is False and seeding_url_text and seeding_url_text[0].strip():
|
||||
self._torrent_seeding_page = seeding_url_text[0].strip()
|
||||
return self._torrent_seeding_page
|
||||
# 首页存在扩展链接,使用扩展链接
|
||||
seeding_url_text = html.xpath('//a[contains(@href,"torrents.php") '
|
||||
'and contains(@href,"seeding")]/@href')
|
||||
if multi_page is False and seeding_url_text and seeding_url_text[0].strip():
|
||||
self._torrent_seeding_page = seeding_url_text[0].strip()
|
||||
return self._torrent_seeding_page
|
||||
|
||||
size_col = 3
|
||||
seeders_col = 4
|
||||
# 搜索size列
|
||||
size_col_xpath = '//tr[position()=1]/' \
|
||||
'td[(img[@class="size"] and img[@alt="size"])' \
|
||||
' or (text() = "大小")' \
|
||||
' or (a/img[@class="size" and @alt="size"])]'
|
||||
if html.xpath(size_col_xpath):
|
||||
size_col = len(html.xpath(f'{size_col_xpath}/preceding-sibling::td')) + 1
|
||||
# 搜索seeders列
|
||||
seeders_col_xpath = '//tr[position()=1]/' \
|
||||
'td[(img[@class="seeders"] and img[@alt="seeders"])' \
|
||||
' or (text() = "在做种")' \
|
||||
' or (a/img[@class="seeders" and @alt="seeders"])]'
|
||||
if html.xpath(seeders_col_xpath):
|
||||
seeders_col = len(html.xpath(f'{seeders_col_xpath}/preceding-sibling::td')) + 1
|
||||
size_col = 3
|
||||
seeders_col = 4
|
||||
# 搜索size列
|
||||
size_col_xpath = '//tr[position()=1]/' \
|
||||
'td[(img[@class="size"] and img[@alt="size"])' \
|
||||
' or (text() = "大小")' \
|
||||
' or (a/img[@class="size" and @alt="size"])]'
|
||||
if html.xpath(size_col_xpath):
|
||||
size_col = len(html.xpath(f'{size_col_xpath}/preceding-sibling::td')) + 1
|
||||
# 搜索seeders列
|
||||
seeders_col_xpath = '//tr[position()=1]/' \
|
||||
'td[(img[@class="seeders"] and img[@alt="seeders"])' \
|
||||
' or (text() = "在做种")' \
|
||||
' or (a/img[@class="seeders" and @alt="seeders"])]'
|
||||
if html.xpath(seeders_col_xpath):
|
||||
seeders_col = len(html.xpath(f'{seeders_col_xpath}/preceding-sibling::td')) + 1
|
||||
|
||||
page_seeding = 0
|
||||
page_seeding_size = 0
|
||||
page_seeding_info = []
|
||||
# 如果 table class="torrents",则增加table[@class="torrents"]
|
||||
table_class = '//table[@class="torrents"]' if html.xpath('//table[@class="torrents"]') else ''
|
||||
seeding_sizes = html.xpath(f'{table_class}//tr[position()>1]/td[{size_col}]')
|
||||
seeding_seeders = html.xpath(f'{table_class}//tr[position()>1]/td[{seeders_col}]/b/a/text()')
|
||||
if not seeding_seeders:
|
||||
seeding_seeders = html.xpath(f'{table_class}//tr[position()>1]/td[{seeders_col}]//text()')
|
||||
if seeding_sizes and seeding_seeders:
|
||||
page_seeding = len(seeding_sizes)
|
||||
page_seeding = 0
|
||||
page_seeding_size = 0
|
||||
page_seeding_info = []
|
||||
# 如果 table class="torrents",则增加table[@class="torrents"]
|
||||
table_class = '//table[@class="torrents"]' if html.xpath('//table[@class="torrents"]') else ''
|
||||
seeding_sizes = html.xpath(f'{table_class}//tr[position()>1]/td[{size_col}]')
|
||||
seeding_seeders = html.xpath(f'{table_class}//tr[position()>1]/td[{seeders_col}]/b/a/text()')
|
||||
if not seeding_seeders:
|
||||
seeding_seeders = html.xpath(f'{table_class}//tr[position()>1]/td[{seeders_col}]//text()')
|
||||
if seeding_sizes and seeding_seeders:
|
||||
page_seeding = len(seeding_sizes)
|
||||
|
||||
for i in range(0, len(seeding_sizes)):
|
||||
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
|
||||
seeders = StringUtils.str_int(seeding_seeders[i])
|
||||
for i in range(0, len(seeding_sizes)):
|
||||
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
|
||||
seeders = StringUtils.str_int(seeding_seeders[i])
|
||||
|
||||
page_seeding_size += size
|
||||
page_seeding_info.append([seeders, size])
|
||||
page_seeding_size += size
|
||||
page_seeding_info.append([seeders, size])
|
||||
|
||||
self.seeding += page_seeding
|
||||
self.seeding_size += page_seeding_size
|
||||
self.seeding_info.extend(page_seeding_info)
|
||||
self.seeding += page_seeding
|
||||
self.seeding_size += page_seeding_size
|
||||
self.seeding_info.extend(page_seeding_info)
|
||||
|
||||
# 是否存在下页数据
|
||||
next_page = None
|
||||
next_page_text = html.xpath(
|
||||
'//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁") or contains(.//text(), ">")]/@href')
|
||||
|
||||
# 防止识别到详情页
|
||||
while next_page_text:
|
||||
next_page = next_page_text.pop().strip()
|
||||
if not next_page.startswith('details.php'):
|
||||
break
|
||||
# 是否存在下页数据
|
||||
next_page = None
|
||||
next_page_text = html.xpath(
|
||||
'//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁") or contains(.//text(), ">")]/@href')
|
||||
|
||||
# fix up page url
|
||||
if next_page:
|
||||
if self.userid not in next_page:
|
||||
next_page = f'{next_page}&userid={self.userid}&type=seeding'
|
||||
# 防止识别到详情页
|
||||
while next_page_text:
|
||||
next_page = next_page_text.pop().strip()
|
||||
if not next_page.startswith('details.php'):
|
||||
break
|
||||
next_page = None
|
||||
|
||||
# fix up page url
|
||||
if next_page:
|
||||
if self.userid not in next_page:
|
||||
next_page = f'{next_page}&userid={self.userid}&type=seeding'
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
return next_page
|
||||
|
||||
@@ -231,57 +248,61 @@ class NexusPhpSiteUserInfo(SiteParserBase):
|
||||
:return:
|
||||
"""
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return
|
||||
|
||||
self._get_user_level(html)
|
||||
self._get_user_level(html)
|
||||
|
||||
self._fixup_traffic_info(html)
|
||||
self._fixup_traffic_info(html)
|
||||
|
||||
# 加入日期
|
||||
join_at_text = html.xpath(
|
||||
'//tr/td[text()="加入日期" or text()="注册日期" or *[text()="加入日期"]]/following-sibling::td[1]//text()'
|
||||
'|//div/b[text()="加入日期"]/../text()')
|
||||
if join_at_text:
|
||||
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].split(' (')[0].strip())
|
||||
# 加入日期
|
||||
join_at_text = html.xpath(
|
||||
'//tr/td[text()="加入日期" or text()="注册日期" or *[text()="加入日期"]]/following-sibling::td[1]//text()'
|
||||
'|//div/b[text()="加入日期"]/../text()')
|
||||
if join_at_text:
|
||||
self.join_at = StringUtils.unify_datetime_str(join_at_text[0].split(' (')[0].strip())
|
||||
|
||||
# 做种体积 & 做种数
|
||||
# seeding 页面获取不到的话,此处再获取一次
|
||||
seeding_sizes = html.xpath('//tr/td[text()="当前上传"]/following-sibling::td[1]//'
|
||||
'table[tr[1][td[4 and text()="尺寸"]]]//tr[position()>1]/td[4]')
|
||||
seeding_seeders = html.xpath('//tr/td[text()="当前上传"]/following-sibling::td[1]//'
|
||||
'table[tr[1][td[5 and text()="做种者"]]]//tr[position()>1]/td[5]//text()')
|
||||
tmp_seeding = len(seeding_sizes)
|
||||
tmp_seeding_size = 0
|
||||
tmp_seeding_info = []
|
||||
for i in range(0, len(seeding_sizes)):
|
||||
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
|
||||
seeders = StringUtils.str_int(seeding_seeders[i])
|
||||
# 做种体积 & 做种数
|
||||
# seeding 页面获取不到的话,此处再获取一次
|
||||
seeding_sizes = html.xpath('//tr/td[text()="当前上传"]/following-sibling::td[1]//'
|
||||
'table[tr[1][td[4 and text()="尺寸"]]]//tr[position()>1]/td[4]')
|
||||
seeding_seeders = html.xpath('//tr/td[text()="当前上传"]/following-sibling::td[1]//'
|
||||
'table[tr[1][td[5 and text()="做种者"]]]//tr[position()>1]/td[5]//text()')
|
||||
tmp_seeding = len(seeding_sizes)
|
||||
tmp_seeding_size = 0
|
||||
tmp_seeding_info = []
|
||||
for i in range(0, len(seeding_sizes)):
|
||||
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
|
||||
seeders = StringUtils.str_int(seeding_seeders[i])
|
||||
|
||||
tmp_seeding_size += size
|
||||
tmp_seeding_info.append([seeders, size])
|
||||
tmp_seeding_size += size
|
||||
tmp_seeding_info.append([seeders, size])
|
||||
|
||||
if not self.seeding_size:
|
||||
self.seeding_size = tmp_seeding_size
|
||||
if not self.seeding:
|
||||
self.seeding = tmp_seeding
|
||||
if not self.seeding_info:
|
||||
self.seeding_info = tmp_seeding_info
|
||||
if not self.seeding_size:
|
||||
self.seeding_size = tmp_seeding_size
|
||||
if not self.seeding:
|
||||
self.seeding = tmp_seeding
|
||||
if not self.seeding_info:
|
||||
self.seeding_info = tmp_seeding_info
|
||||
|
||||
seeding_sizes = html.xpath('//tr/td[text()="做种统计"]/following-sibling::td[1]//text()')
|
||||
if seeding_sizes:
|
||||
seeding_match = re.search(r"总做种数:\s+(\d+)", seeding_sizes[0], re.IGNORECASE)
|
||||
seeding_size_match = re.search(r"总做种体积:\s+([\d,.\s]+[KMGTPI]*B)", seeding_sizes[0], re.IGNORECASE)
|
||||
tmp_seeding = StringUtils.str_int(seeding_match.group(1)) if (
|
||||
seeding_match and seeding_match.group(1)) else 0
|
||||
tmp_seeding_size = StringUtils.num_filesize(
|
||||
seeding_size_match.group(1).strip()) if seeding_size_match else 0
|
||||
if not self.seeding_size:
|
||||
self.seeding_size = tmp_seeding_size
|
||||
if not self.seeding:
|
||||
self.seeding = tmp_seeding
|
||||
seeding_sizes = html.xpath('//tr/td[text()="做种统计"]/following-sibling::td[1]//text()')
|
||||
if seeding_sizes:
|
||||
seeding_match = re.search(r"总做种数:\s+(\d+)", seeding_sizes[0], re.IGNORECASE)
|
||||
seeding_size_match = re.search(r"总做种体积:\s+([\d,.\s]+[KMGTPI]*B)", seeding_sizes[0], re.IGNORECASE)
|
||||
tmp_seeding = StringUtils.str_int(seeding_match.group(1)) if (
|
||||
seeding_match and seeding_match.group(1)) else 0
|
||||
tmp_seeding_size = StringUtils.num_filesize(
|
||||
seeding_size_match.group(1).strip()) if seeding_size_match else 0
|
||||
if not self.seeding_size:
|
||||
self.seeding_size = tmp_seeding_size
|
||||
if not self.seeding:
|
||||
self.seeding = tmp_seeding
|
||||
|
||||
self._fixup_torrent_seeding_page(html)
|
||||
self._fixup_torrent_seeding_page(html)
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _fixup_torrent_seeding_page(self, html):
|
||||
"""
|
||||
@@ -348,43 +369,51 @@ class NexusPhpSiteUserInfo(SiteParserBase):
|
||||
|
||||
def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]:
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
|
||||
message_links = html.xpath('//tr[not(./td/img[@alt="Read"])]/td/a[contains(@href, "viewmessage")]/@href')
|
||||
msg_links.extend(message_links)
|
||||
# 是否存在下页数据
|
||||
next_page = None
|
||||
next_page_text = html.xpath('//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁")]/@href')
|
||||
if next_page_text:
|
||||
next_page = next_page_text[-1].strip()
|
||||
message_links = html.xpath('//tr[not(./td/img[@alt="Read"])]/td/a[contains(@href, "viewmessage")]/@href')
|
||||
msg_links.extend(message_links)
|
||||
# 是否存在下页数据
|
||||
next_page = None
|
||||
next_page_text = html.xpath('//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁")]/@href')
|
||||
if next_page_text:
|
||||
next_page = next_page_text[-1].strip()
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
return next_page
|
||||
|
||||
def _parse_message_content(self, html_text):
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None, None, None
|
||||
# 标题
|
||||
message_head_text = None
|
||||
message_head = html.xpath('//h1/text()'
|
||||
'|//div[@class="layui-card-header"]/span[1]/text()')
|
||||
if message_head:
|
||||
message_head_text = message_head[-1].strip()
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None, None, None
|
||||
# 标题
|
||||
message_head_text = None
|
||||
message_head = html.xpath('//h1/text()'
|
||||
'|//div[@class="layui-card-header"]/span[1]/text()')
|
||||
if message_head:
|
||||
message_head_text = message_head[-1].strip()
|
||||
|
||||
# 消息时间
|
||||
message_date_text = None
|
||||
message_date = html.xpath('//h1/following-sibling::table[.//tr/td[@class="colhead"]]//tr[2]/td[2]'
|
||||
'|//div[@class="layui-card-header"]/span[2]/span[2]')
|
||||
if message_date:
|
||||
message_date_text = message_date[0].xpath("string(.)").strip()
|
||||
# 消息时间
|
||||
message_date_text = None
|
||||
message_date = html.xpath('//h1/following-sibling::table[.//tr/td[@class="colhead"]]//tr[2]/td[2]'
|
||||
'|//div[@class="layui-card-header"]/span[2]/span[2]')
|
||||
if message_date:
|
||||
message_date_text = message_date[0].xpath("string(.)").strip()
|
||||
|
||||
# 消息内容
|
||||
message_content_text = None
|
||||
message_content = html.xpath('//h1/following-sibling::table[.//tr/td[@class="colhead"]]//tr[3]/td'
|
||||
'|//div[contains(@class,"layui-card-body")]')
|
||||
if message_content:
|
||||
message_content_text = message_content[0].xpath("string(.)").strip()
|
||||
# 消息内容
|
||||
message_content_text = None
|
||||
message_content = html.xpath('//h1/following-sibling::table[.//tr/td[@class="colhead"]]//tr[3]/td'
|
||||
'|//div[contains(@class,"layui-card-body")]')
|
||||
if message_content:
|
||||
message_content_text = message_content[0].xpath("string(.)").strip()
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
return message_head_text, message_date_text, message_content_text
|
||||
|
||||
|
||||
@@ -114,48 +114,56 @@ class NexusRabbitSiteUserInfo(SiteParserBase):
|
||||
def _parse_user_base_info(self, html_text: str):
|
||||
"""只有奶糖余额才需要在 base 中获取,其它均可以在详情页拿到"""
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return
|
||||
bonus = html.xpath(
|
||||
'//div[contains(text(), "奶糖余额")]/following-sibling::div[1]/text()'
|
||||
)
|
||||
if bonus:
|
||||
self.bonus = StringUtils.str_float(bonus[0].strip())
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return
|
||||
bonus = html.xpath(
|
||||
'//div[contains(text(), "奶糖余额")]/following-sibling::div[1]/text()'
|
||||
)
|
||||
if bonus:
|
||||
self.bonus = StringUtils.str_float(bonus[0].strip())
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_user_detail_info(self, html_text: str):
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return
|
||||
# 缩小一下查找范围,所有的信息都在这个 div 里
|
||||
user_info = html.xpath('//div[contains(@class, "layui-hares-user-info-right")]')
|
||||
if not user_info:
|
||||
return
|
||||
user_info = user_info[0]
|
||||
# 用户名
|
||||
if username := user_info.xpath(
|
||||
'.//span[contains(text(), "用户名")]/a/span/text()'
|
||||
):
|
||||
self.username = username[0].strip()
|
||||
# 等级
|
||||
if user_level := user_info.xpath('.//span[contains(text(), "等级")]/b/text()'):
|
||||
self.user_level = user_level[0].strip()
|
||||
# 加入日期
|
||||
if join_date := user_info.xpath('.//span[contains(text(), "注册日期")]/text()'):
|
||||
join_date = join_date[0].strip().split("\r")[0].removeprefix("注册日期:")
|
||||
self.join_at = StringUtils.unify_datetime_str(join_date)
|
||||
# 上传量
|
||||
if upload := user_info.xpath('.//span[contains(text(), "上传量")]/text()'):
|
||||
self.upload = StringUtils.num_filesize(
|
||||
upload[0].strip().removeprefix("上传量:")
|
||||
)
|
||||
# 下载量
|
||||
if download := user_info.xpath('.//span[contains(text(), "下载量")]/text()'):
|
||||
self.download = StringUtils.num_filesize(
|
||||
download[0].strip().removeprefix("下载量:")
|
||||
)
|
||||
# 分享率
|
||||
if ratio := user_info.xpath('.//span[contains(text(), "分享率")]/em/text()'):
|
||||
self.ratio = StringUtils.str_float(ratio[0].strip())
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return
|
||||
# 缩小一下查找范围,所有的信息都在这个 div 里
|
||||
user_info = html.xpath('//div[contains(@class, "layui-hares-user-info-right")]')
|
||||
if not user_info:
|
||||
return
|
||||
user_info = user_info[0]
|
||||
# 用户名
|
||||
if username := user_info.xpath(
|
||||
'.//span[contains(text(), "用户名")]/a/span/text()'
|
||||
):
|
||||
self.username = username[0].strip()
|
||||
# 等级
|
||||
if user_level := user_info.xpath('.//span[contains(text(), "等级")]/b/text()'):
|
||||
self.user_level = user_level[0].strip()
|
||||
# 加入日期
|
||||
if join_date := user_info.xpath('.//span[contains(text(), "注册日期")]/text()'):
|
||||
join_date = join_date[0].strip().split("\r")[0].removeprefix("注册日期:")
|
||||
self.join_at = StringUtils.unify_datetime_str(join_date)
|
||||
# 上传量
|
||||
if upload := user_info.xpath('.//span[contains(text(), "上传量")]/text()'):
|
||||
self.upload = StringUtils.num_filesize(
|
||||
upload[0].strip().removeprefix("上传量:")
|
||||
)
|
||||
# 下载量
|
||||
if download := user_info.xpath('.//span[contains(text(), "下载量")]/text()'):
|
||||
self.download = StringUtils.num_filesize(
|
||||
download[0].strip().removeprefix("下载量:")
|
||||
)
|
||||
# 分享率
|
||||
if ratio := user_info.xpath('.//span[contains(text(), "分享率")]/em/text()'):
|
||||
self.ratio = StringUtils.str_float(ratio[0].strip())
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_message_content(self, html_text):
|
||||
"""
|
||||
|
||||
@@ -24,9 +24,13 @@ class SmallHorseSiteUserInfo(SiteParserBase):
|
||||
def _parse_user_base_info(self, html_text: str):
|
||||
html_text = self._prepare_html_text(html_text)
|
||||
html = etree.HTML(html_text)
|
||||
ret = html.xpath('//a[contains(@href, "user.php")]//text()')
|
||||
if ret:
|
||||
self.username = str(ret[0])
|
||||
try:
|
||||
ret = html.xpath('//a[contains(@href, "user.php")]//text()')
|
||||
if ret:
|
||||
self.username = str(ret[0])
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_user_traffic_info(self, html_text: str):
|
||||
"""
|
||||
@@ -36,21 +40,25 @@ class SmallHorseSiteUserInfo(SiteParserBase):
|
||||
"""
|
||||
html_text = self._prepare_html_text(html_text)
|
||||
html = etree.HTML(html_text)
|
||||
tmps = html.xpath('//ul[@class = "stats nobullet"]')
|
||||
if tmps:
|
||||
if tmps[1].xpath("li") and tmps[1].xpath("li")[0].xpath("span//text()"):
|
||||
self.join_at = StringUtils.unify_datetime_str(tmps[1].xpath("li")[0].xpath("span//text()")[0])
|
||||
self.upload = StringUtils.num_filesize(str(tmps[1].xpath("li")[2].xpath("text()")[0]).split(":")[1].strip())
|
||||
self.download = StringUtils.num_filesize(
|
||||
str(tmps[1].xpath("li")[3].xpath("text()")[0]).split(":")[1].strip())
|
||||
if tmps[1].xpath("li")[4].xpath("span//text()"):
|
||||
self.ratio = StringUtils.str_float(str(tmps[1].xpath("li")[4].xpath("span//text()")[0]).replace('∞', '0'))
|
||||
else:
|
||||
self.ratio = StringUtils.str_float(str(tmps[1].xpath("li")[5].xpath("text()")[0]).split(":")[1])
|
||||
self.bonus = StringUtils.str_float(str(tmps[1].xpath("li")[5].xpath("text()")[0]).split(":")[1])
|
||||
self.user_level = str(tmps[3].xpath("li")[0].xpath("text()")[0]).split(":")[1].strip()
|
||||
self.leeching = StringUtils.str_int(
|
||||
(tmps[4].xpath("li")[6].xpath("text()")[0]).split(":")[1].replace("[", ""))
|
||||
try:
|
||||
tmps = html.xpath('//ul[@class = "stats nobullet"]')
|
||||
if tmps:
|
||||
if tmps[1].xpath("li") and tmps[1].xpath("li")[0].xpath("span//text()"):
|
||||
self.join_at = StringUtils.unify_datetime_str(tmps[1].xpath("li")[0].xpath("span//text()")[0])
|
||||
self.upload = StringUtils.num_filesize(str(tmps[1].xpath("li")[2].xpath("text()")[0]).split(":")[1].strip())
|
||||
self.download = StringUtils.num_filesize(
|
||||
str(tmps[1].xpath("li")[3].xpath("text()")[0]).split(":")[1].strip())
|
||||
if tmps[1].xpath("li")[4].xpath("span//text()"):
|
||||
self.ratio = StringUtils.str_float(str(tmps[1].xpath("li")[4].xpath("span//text()")[0]).replace('∞', '0'))
|
||||
else:
|
||||
self.ratio = StringUtils.str_float(str(tmps[1].xpath("li")[5].xpath("text()")[0]).split(":")[1])
|
||||
self.bonus = StringUtils.str_float(str(tmps[1].xpath("li")[5].xpath("text()")[0]).split(":")[1])
|
||||
self.user_level = str(tmps[3].xpath("li")[0].xpath("text()")[0]).split(":")[1].strip()
|
||||
self.leeching = StringUtils.str_int(
|
||||
(tmps[4].xpath("li")[6].xpath("text()")[0]).split(":")[1].replace("[", ""))
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_user_detail_info(self, html_text: str):
|
||||
pass
|
||||
@@ -63,39 +71,42 @@ class SmallHorseSiteUserInfo(SiteParserBase):
|
||||
:return: 下页地址
|
||||
"""
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
|
||||
size_col = 6
|
||||
seeders_col = 8
|
||||
size_col = 6
|
||||
seeders_col = 8
|
||||
|
||||
page_seeding = 0
|
||||
page_seeding_size = 0
|
||||
page_seeding_info = []
|
||||
seeding_sizes = html.xpath(f'//table[@id="torrent_table"]//tr[position()>1]/td[{size_col}]')
|
||||
seeding_seeders = html.xpath(f'//table[@id="torrent_table"]//tr[position()>1]/td[{seeders_col}]')
|
||||
if seeding_sizes and seeding_seeders:
|
||||
page_seeding = len(seeding_sizes)
|
||||
page_seeding = 0
|
||||
page_seeding_size = 0
|
||||
page_seeding_info = []
|
||||
seeding_sizes = html.xpath(f'//table[@id="torrent_table"]//tr[position()>1]/td[{size_col}]')
|
||||
seeding_seeders = html.xpath(f'//table[@id="torrent_table"]//tr[position()>1]/td[{seeders_col}]')
|
||||
if seeding_sizes and seeding_seeders:
|
||||
page_seeding = len(seeding_sizes)
|
||||
|
||||
for i in range(0, len(seeding_sizes)):
|
||||
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
|
||||
seeders = StringUtils.str_int(seeding_seeders[i].xpath("string(.)").strip())
|
||||
for i in range(0, len(seeding_sizes)):
|
||||
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
|
||||
seeders = StringUtils.str_int(seeding_seeders[i].xpath("string(.)").strip())
|
||||
|
||||
page_seeding_size += size
|
||||
page_seeding_info.append([seeders, size])
|
||||
page_seeding_size += size
|
||||
page_seeding_info.append([seeders, size])
|
||||
|
||||
self.seeding += page_seeding
|
||||
self.seeding_size += page_seeding_size
|
||||
self.seeding_info.extend(page_seeding_info)
|
||||
|
||||
# 是否存在下页数据
|
||||
next_page = None
|
||||
next_pages = html.xpath('//ul[@class="pagination"]/li[contains(@class,"active")]/following-sibling::li')
|
||||
if next_pages and len(next_pages) > 1:
|
||||
page_num = next_pages[0].xpath("string(.)").strip()
|
||||
if page_num.isdigit():
|
||||
next_page = f"{self._torrent_seeding_page}&page={page_num}"
|
||||
self.seeding += page_seeding
|
||||
self.seeding_size += page_seeding_size
|
||||
self.seeding_info.extend(page_seeding_info)
|
||||
|
||||
# 是否存在下页数据
|
||||
next_page = None
|
||||
next_pages = html.xpath('//ul[@class="pagination"]/li[contains(@class,"active")]/following-sibling::li')
|
||||
if next_pages and len(next_pages) > 1:
|
||||
page_num = next_pages[0].xpath("string(.)").strip()
|
||||
if page_num.isdigit():
|
||||
next_page = f"{self._torrent_seeding_page}&page={page_num}"
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
return next_page
|
||||
|
||||
def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]:
|
||||
|
||||
@@ -32,29 +32,33 @@ class TorrentLeechSiteUserInfo(SiteParserBase):
|
||||
"""
|
||||
html_text = self._prepare_html_text(html_text)
|
||||
html = etree.HTML(html_text)
|
||||
upload_html = html.xpath('//div[contains(@class,"profile-uploaded")]//span/text()')
|
||||
if upload_html:
|
||||
self.upload = StringUtils.num_filesize(upload_html[0])
|
||||
download_html = html.xpath('//div[contains(@class,"profile-downloaded")]//span/text()')
|
||||
if download_html:
|
||||
self.download = StringUtils.num_filesize(download_html[0])
|
||||
ratio_html = html.xpath('//div[contains(@class,"profile-ratio")]//span/text()')
|
||||
if ratio_html:
|
||||
self.ratio = StringUtils.str_float(ratio_html[0].replace('∞', '0'))
|
||||
try:
|
||||
upload_html = html.xpath('//div[contains(@class,"profile-uploaded")]//span/text()')
|
||||
if upload_html:
|
||||
self.upload = StringUtils.num_filesize(upload_html[0])
|
||||
download_html = html.xpath('//div[contains(@class,"profile-downloaded")]//span/text()')
|
||||
if download_html:
|
||||
self.download = StringUtils.num_filesize(download_html[0])
|
||||
ratio_html = html.xpath('//div[contains(@class,"profile-ratio")]//span/text()')
|
||||
if ratio_html:
|
||||
self.ratio = StringUtils.str_float(ratio_html[0].replace('∞', '0'))
|
||||
|
||||
user_level_html = html.xpath('//table[contains(@class, "profileViewTable")]'
|
||||
'//tr/td[text()="Class"]/following-sibling::td/text()')
|
||||
if user_level_html:
|
||||
self.user_level = user_level_html[0].strip()
|
||||
user_level_html = html.xpath('//table[contains(@class, "profileViewTable")]'
|
||||
'//tr/td[text()="Class"]/following-sibling::td/text()')
|
||||
if user_level_html:
|
||||
self.user_level = user_level_html[0].strip()
|
||||
|
||||
join_at_html = html.xpath('//table[contains(@class, "profileViewTable")]'
|
||||
'//tr/td[text()="Registration date"]/following-sibling::td/text()')
|
||||
if join_at_html:
|
||||
self.join_at = StringUtils.unify_datetime_str(join_at_html[0].strip())
|
||||
join_at_html = html.xpath('//table[contains(@class, "profileViewTable")]'
|
||||
'//tr/td[text()="Registration date"]/following-sibling::td/text()')
|
||||
if join_at_html:
|
||||
self.join_at = StringUtils.unify_datetime_str(join_at_html[0].strip())
|
||||
|
||||
bonus_html = html.xpath('//span[contains(@class, "total-TL-points")]/text()')
|
||||
if bonus_html:
|
||||
self.bonus = StringUtils.str_float(bonus_html[0].strip())
|
||||
bonus_html = html.xpath('//span[contains(@class, "total-TL-points")]/text()')
|
||||
if bonus_html:
|
||||
self.bonus = StringUtils.str_float(bonus_html[0].strip())
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_user_detail_info(self, html_text: str):
|
||||
pass
|
||||
@@ -67,33 +71,37 @@ class TorrentLeechSiteUserInfo(SiteParserBase):
|
||||
:return: 下页地址
|
||||
"""
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
|
||||
size_col = 2
|
||||
seeders_col = 7
|
||||
size_col = 2
|
||||
seeders_col = 7
|
||||
|
||||
page_seeding = 0
|
||||
page_seeding_size = 0
|
||||
page_seeding_info = []
|
||||
seeding_sizes = html.xpath(f'//tbody/tr/td[{size_col}]')
|
||||
seeding_seeders = html.xpath(f'//tbody/tr/td[{seeders_col}]/text()')
|
||||
if seeding_sizes and seeding_seeders:
|
||||
page_seeding = len(seeding_sizes)
|
||||
page_seeding = 0
|
||||
page_seeding_size = 0
|
||||
page_seeding_info = []
|
||||
seeding_sizes = html.xpath(f'//tbody/tr/td[{size_col}]')
|
||||
seeding_seeders = html.xpath(f'//tbody/tr/td[{seeders_col}]/text()')
|
||||
if seeding_sizes and seeding_seeders:
|
||||
page_seeding = len(seeding_sizes)
|
||||
|
||||
for i in range(0, len(seeding_sizes)):
|
||||
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
|
||||
seeders = StringUtils.str_int(seeding_seeders[i])
|
||||
for i in range(0, len(seeding_sizes)):
|
||||
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
|
||||
seeders = StringUtils.str_int(seeding_seeders[i])
|
||||
|
||||
page_seeding_size += size
|
||||
page_seeding_info.append([seeders, size])
|
||||
page_seeding_size += size
|
||||
page_seeding_info.append([seeders, size])
|
||||
|
||||
self.seeding += page_seeding
|
||||
self.seeding_size += page_seeding_size
|
||||
self.seeding_info.extend(page_seeding_info)
|
||||
self.seeding += page_seeding
|
||||
self.seeding_size += page_seeding_size
|
||||
self.seeding_info.extend(page_seeding_info)
|
||||
|
||||
# 是否存在下页数据
|
||||
next_page = None
|
||||
# 是否存在下页数据
|
||||
next_page = None
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
return next_page
|
||||
|
||||
|
||||
@@ -14,21 +14,24 @@ class Unit3dSiteUserInfo(SiteParserBase):
|
||||
def _parse_user_base_info(self, html_text: str):
|
||||
html_text = self._prepare_html_text(html_text)
|
||||
html = etree.HTML(html_text)
|
||||
try:
|
||||
tmps = html.xpath('//a[contains(@href, "/users/") and contains(@href, "settings")]/@href')
|
||||
if tmps:
|
||||
user_name_match = re.search(r"/users/(.+)/settings", tmps[0])
|
||||
if user_name_match and user_name_match.group().strip():
|
||||
self.username = user_name_match.group(1)
|
||||
self._torrent_seeding_page = f"/users/{self.username}/active?perPage=100&client=&seeding=include"
|
||||
self._user_detail_page = f"/users/{self.username}"
|
||||
|
||||
tmps = html.xpath('//a[contains(@href, "/users/") and contains(@href, "settings")]/@href')
|
||||
if tmps:
|
||||
user_name_match = re.search(r"/users/(.+)/settings", tmps[0])
|
||||
if user_name_match and user_name_match.group().strip():
|
||||
self.username = user_name_match.group(1)
|
||||
self._torrent_seeding_page = f"/users/{self.username}/active?perPage=100&client=&seeding=include"
|
||||
self._user_detail_page = f"/users/{self.username}"
|
||||
|
||||
tmps = html.xpath('//a[contains(@href, "bonus/earnings")]')
|
||||
if tmps:
|
||||
bonus_text = tmps[0].xpath("string(.)")
|
||||
bonus_match = re.search(r"([\d,.]+)", bonus_text)
|
||||
if bonus_match and bonus_match.group(1).strip():
|
||||
self.bonus = StringUtils.str_float(bonus_match.group(1))
|
||||
tmps = html.xpath('//a[contains(@href, "bonus/earnings")]')
|
||||
if tmps:
|
||||
bonus_text = tmps[0].xpath("string(.)")
|
||||
bonus_match = re.search(r"([\d,.]+)", bonus_text)
|
||||
if bonus_match and bonus_match.group(1).strip():
|
||||
self.bonus = StringUtils.str_float(bonus_match.group(1))
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_site_page(self, html_text: str):
|
||||
pass
|
||||
@@ -40,21 +43,25 @@ class Unit3dSiteUserInfo(SiteParserBase):
|
||||
:return:
|
||||
"""
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
|
||||
# 用户等级
|
||||
user_levels_text = html.xpath('//div[contains(@class, "content")]//span[contains(@class, "badge-user")]/text()')
|
||||
if user_levels_text:
|
||||
self.user_level = user_levels_text[0].strip()
|
||||
# 用户等级
|
||||
user_levels_text = html.xpath('//div[contains(@class, "content")]//span[contains(@class, "badge-user")]/text()')
|
||||
if user_levels_text:
|
||||
self.user_level = user_levels_text[0].strip()
|
||||
|
||||
# 加入日期
|
||||
join_at_text = html.xpath('//div[contains(@class, "content")]//h4[contains(text(), "注册日期") '
|
||||
'or contains(text(), "註冊日期") '
|
||||
'or contains(text(), "Registration date")]/text()')
|
||||
if join_at_text:
|
||||
self.join_at = StringUtils.unify_datetime_str(
|
||||
join_at_text[0].replace('注册日期', '').replace('註冊日期', '').replace('Registration date', ''))
|
||||
# 加入日期
|
||||
join_at_text = html.xpath('//div[contains(@class, "content")]//h4[contains(text(), "注册日期") '
|
||||
'or contains(text(), "註冊日期") '
|
||||
'or contains(text(), "Registration date")]/text()')
|
||||
if join_at_text:
|
||||
self.join_at = StringUtils.unify_datetime_str(
|
||||
join_at_text[0].replace('注册日期', '').replace('註冊日期', '').replace('Registration date', ''))
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: Optional[bool] = False) -> Optional[str]:
|
||||
"""
|
||||
@@ -64,44 +71,48 @@ class Unit3dSiteUserInfo(SiteParserBase):
|
||||
:return: 下页地址
|
||||
"""
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return None
|
||||
|
||||
size_col = 9
|
||||
seeders_col = 2
|
||||
# 搜索size列
|
||||
if html.xpath('//thead//th[contains(@class,"size")]'):
|
||||
size_col = len(html.xpath('//thead//th[contains(@class,"size")][1]/preceding-sibling::th')) + 1
|
||||
# 搜索seeders列
|
||||
if html.xpath('//thead//th[contains(@class,"seeders")]'):
|
||||
seeders_col = len(html.xpath('//thead//th[contains(@class,"seeders")]/preceding-sibling::th')) + 1
|
||||
size_col = 9
|
||||
seeders_col = 2
|
||||
# 搜索size列
|
||||
if html.xpath('//thead//th[contains(@class,"size")]'):
|
||||
size_col = len(html.xpath('//thead//th[contains(@class,"size")][1]/preceding-sibling::th')) + 1
|
||||
# 搜索seeders列
|
||||
if html.xpath('//thead//th[contains(@class,"seeders")]'):
|
||||
seeders_col = len(html.xpath('//thead//th[contains(@class,"seeders")]/preceding-sibling::th')) + 1
|
||||
|
||||
page_seeding = 0
|
||||
page_seeding_size = 0
|
||||
page_seeding_info = []
|
||||
seeding_sizes = html.xpath(f'//tr[position()]/td[{size_col}]')
|
||||
seeding_seeders = html.xpath(f'//tr[position()]/td[{seeders_col}]')
|
||||
if seeding_sizes and seeding_seeders:
|
||||
page_seeding = len(seeding_sizes)
|
||||
page_seeding = 0
|
||||
page_seeding_size = 0
|
||||
page_seeding_info = []
|
||||
seeding_sizes = html.xpath(f'//tr[position()]/td[{size_col}]')
|
||||
seeding_seeders = html.xpath(f'//tr[position()]/td[{seeders_col}]')
|
||||
if seeding_sizes and seeding_seeders:
|
||||
page_seeding = len(seeding_sizes)
|
||||
|
||||
for i in range(0, len(seeding_sizes)):
|
||||
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
|
||||
seeders = StringUtils.str_int(seeding_seeders[i].xpath("string(.)").strip())
|
||||
for i in range(0, len(seeding_sizes)):
|
||||
size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip())
|
||||
seeders = StringUtils.str_int(seeding_seeders[i].xpath("string(.)").strip())
|
||||
|
||||
page_seeding_size += size
|
||||
page_seeding_info.append([seeders, size])
|
||||
page_seeding_size += size
|
||||
page_seeding_info.append([seeders, size])
|
||||
|
||||
self.seeding += page_seeding
|
||||
self.seeding_size += page_seeding_size
|
||||
self.seeding_info.extend(page_seeding_info)
|
||||
self.seeding += page_seeding
|
||||
self.seeding_size += page_seeding_size
|
||||
self.seeding_info.extend(page_seeding_info)
|
||||
|
||||
# 是否存在下页数据
|
||||
next_page = None
|
||||
next_pages = html.xpath('//ul[@class="pagination"]/li[contains(@class,"active")]/following-sibling::li')
|
||||
if next_pages and len(next_pages) > 1:
|
||||
page_num = next_pages[0].xpath("string(.)").strip()
|
||||
if page_num.isdigit():
|
||||
next_page = f"{self._torrent_seeding_page}&page={page_num}"
|
||||
# 是否存在下页数据
|
||||
next_page = None
|
||||
next_pages = html.xpath('//ul[@class="pagination"]/li[contains(@class,"active")]/following-sibling::li')
|
||||
if next_pages and len(next_pages) > 1:
|
||||
page_num = next_pages[0].xpath("string(.)").strip()
|
||||
if page_num.isdigit():
|
||||
next_page = f"{self._torrent_seeding_page}&page={page_num}"
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
return next_page
|
||||
|
||||
|
||||
@@ -0,0 +1,705 @@
|
||||
import datetime
|
||||
import re
|
||||
import traceback
|
||||
from typing import Any, Optional
|
||||
from typing import List
|
||||
from urllib.parse import quote, urlencode, urlparse, parse_qs
|
||||
|
||||
from jinja2 import Template
|
||||
from pyquery import PyQuery
|
||||
|
||||
from app.core.config import settings
|
||||
from app.log import logger
|
||||
from app.schemas.types import MediaType
|
||||
from app.utils.http import RequestUtils
|
||||
from app.utils.string import StringUtils
|
||||
|
||||
|
||||
class SiteSpider:
|
||||
"""
|
||||
站点爬虫
|
||||
"""
|
||||
|
||||
@property
|
||||
def __class__(self):
|
||||
return object
|
||||
|
||||
@property
|
||||
def __dict__(self):
|
||||
return {}
|
||||
|
||||
@property
|
||||
def __dir__(self):
|
||||
raise AttributeError(f"Cannot read protected attribute!")
|
||||
|
||||
def __init__(self,
|
||||
indexer: dict,
|
||||
keyword: Optional[str] = None,
|
||||
mtype: MediaType = None,
|
||||
cat: Optional[str] = None,
|
||||
page: Optional[int] = 0,
|
||||
referer: Optional[str] = None):
|
||||
"""
|
||||
设置查询参数
|
||||
:param indexer: 索引器
|
||||
:param keyword: 搜索关键字,如果数组则为批量搜索
|
||||
:param mtype: 媒体类型
|
||||
:param cat: 搜索分类
|
||||
:param page: 页码
|
||||
:param referer: Referer
|
||||
"""
|
||||
if not indexer:
|
||||
return
|
||||
self.keyword = keyword
|
||||
self.cat = cat
|
||||
self.mtype = mtype
|
||||
self.indexerid = indexer.get('id')
|
||||
self.indexername = indexer.get('name')
|
||||
self.search = indexer.get('search')
|
||||
self.batch = indexer.get('batch')
|
||||
self.browse = indexer.get('browse')
|
||||
self.category = indexer.get('category')
|
||||
self.list = indexer.get('torrents').get('list', {})
|
||||
self.fields = indexer.get('torrents').get('fields')
|
||||
if not keyword and self.browse:
|
||||
self.list = self.browse.get('list') or self.list
|
||||
self.fields = self.browse.get('fields') or self.fields
|
||||
self.domain = indexer.get('domain')
|
||||
self.result_num = int(indexer.get('result_num') or 100)
|
||||
self._timeout = int(indexer.get('timeout') or 15)
|
||||
self.page = page
|
||||
if self.domain and not str(self.domain).endswith("/"):
|
||||
self.domain = self.domain + "/"
|
||||
self.ua = indexer.get('ua') or settings.USER_AGENT
|
||||
self.proxies = settings.PROXY if indexer.get('proxy') else None
|
||||
self.proxy_server = settings.PROXY_SERVER if indexer.get('proxy') else None
|
||||
self.cookie = indexer.get('cookie')
|
||||
self.referer = referer
|
||||
# 初始化属性
|
||||
self.is_error = False
|
||||
self.torrents_info = {}
|
||||
self.torrents_info_array = []
|
||||
|
||||
def get_torrents(self) -> List[dict]:
|
||||
"""
|
||||
开始请求
|
||||
"""
|
||||
if not self.search or not self.domain:
|
||||
return []
|
||||
|
||||
# 种子搜索相对路径
|
||||
paths = self.search.get('paths', [])
|
||||
torrentspath = ""
|
||||
if len(paths) == 1:
|
||||
torrentspath = paths[0].get('path', '')
|
||||
else:
|
||||
for path in paths:
|
||||
if path.get("type") == "all" and not self.mtype:
|
||||
torrentspath = path.get('path')
|
||||
break
|
||||
elif path.get("type") == "movie" and self.mtype == MediaType.MOVIE:
|
||||
torrentspath = path.get('path')
|
||||
break
|
||||
elif path.get("type") == "tv" and self.mtype == MediaType.TV:
|
||||
torrentspath = path.get('path')
|
||||
break
|
||||
|
||||
# 精确搜索
|
||||
if self.keyword:
|
||||
if isinstance(self.keyword, list):
|
||||
# 批量查询
|
||||
if self.batch:
|
||||
delimiter = self.batch.get('delimiter') or ' '
|
||||
space_replace = self.batch.get('space_replace') or ' '
|
||||
search_word = delimiter.join([str(k).replace(' ',
|
||||
space_replace) for k in self.keyword])
|
||||
else:
|
||||
search_word = " ".join(self.keyword)
|
||||
# 查询模式:或
|
||||
search_mode = "1"
|
||||
else:
|
||||
# 单个查询
|
||||
search_word = self.keyword
|
||||
# 查询模式与
|
||||
search_mode = "0"
|
||||
|
||||
# 搜索URL
|
||||
indexer_params = self.search.get("params", {}).copy()
|
||||
if indexer_params:
|
||||
search_area = indexer_params.get('search_area')
|
||||
# search_area非0表示支持imdbid搜索
|
||||
if (search_area and
|
||||
(not self.keyword or not self.keyword.startswith('tt'))):
|
||||
# 支持imdbid搜索,但关键字不是imdbid时,不启用imdbid搜索
|
||||
indexer_params.pop('search_area')
|
||||
# 变量字典
|
||||
inputs_dict = {
|
||||
"keyword": search_word
|
||||
}
|
||||
# 查询参数,默认查询标题
|
||||
params = {
|
||||
"search_mode": search_mode,
|
||||
"search_area": 0,
|
||||
"page": self.page or 0,
|
||||
"notnewword": 1
|
||||
}
|
||||
# 额外参数
|
||||
for key, value in indexer_params.items():
|
||||
params.update({
|
||||
"%s" % key: str(value).format(**inputs_dict)
|
||||
})
|
||||
# 分类条件
|
||||
if self.category:
|
||||
if self.mtype == MediaType.TV:
|
||||
cats = self.category.get("tv") or []
|
||||
elif self.mtype == MediaType.MOVIE:
|
||||
cats = self.category.get("movie") or []
|
||||
else:
|
||||
cats = (self.category.get("movie") or []) + (self.category.get("tv") or [])
|
||||
allowed_cats = set(self.cat.split(',')) if self.cat else None
|
||||
for cat in cats:
|
||||
if allowed_cats and str(cat.get('id')) not in allowed_cats:
|
||||
continue
|
||||
if self.category.get("field"):
|
||||
value = params.get(self.category.get("field"), "")
|
||||
params.update({
|
||||
"%s" % self.category.get("field"): value + self.category.get("delimiter",
|
||||
' ') + cat.get("id")
|
||||
})
|
||||
else:
|
||||
params.update({
|
||||
"cat%s" % cat.get("id"): 1
|
||||
})
|
||||
searchurl = self.domain + torrentspath + "?" + urlencode(params)
|
||||
else:
|
||||
# 变量字典
|
||||
inputs_dict = {
|
||||
"keyword": quote(search_word),
|
||||
"page": self.page or 0
|
||||
}
|
||||
# 无额外参数
|
||||
searchurl = self.domain + str(torrentspath).format(**inputs_dict)
|
||||
|
||||
# 列表浏览
|
||||
else:
|
||||
# 变量字典
|
||||
inputs_dict = {
|
||||
"page": self.page or 0,
|
||||
"keyword": ""
|
||||
}
|
||||
# 有单独浏览路径
|
||||
if self.browse:
|
||||
torrentspath = self.browse.get("path")
|
||||
if self.browse.get("start"):
|
||||
start_page = int(self.browse.get("start")) + int(self.page or 0)
|
||||
inputs_dict.update({
|
||||
"page": start_page
|
||||
})
|
||||
elif self.page:
|
||||
torrentspath = torrentspath + f"?page={self.page}"
|
||||
# 搜索Url
|
||||
searchurl = self.domain + str(torrentspath).format(**inputs_dict)
|
||||
|
||||
logger.info(f"开始请求:{searchurl}")
|
||||
|
||||
# requests请求
|
||||
ret = RequestUtils(
|
||||
ua=self.ua,
|
||||
cookies=self.cookie,
|
||||
timeout=self._timeout,
|
||||
referer=self.referer,
|
||||
proxies=self.proxies
|
||||
).get_res(searchurl, allow_redirects=True)
|
||||
# 解析返回
|
||||
return self.parse(
|
||||
RequestUtils.get_decoded_html_content(
|
||||
ret,
|
||||
performance_mode=settings.ENCODING_DETECTION_PERFORMANCE_MODE,
|
||||
confidence_threshold=settings.ENCODING_DETECTION_MIN_CONFIDENCE
|
||||
)
|
||||
)
|
||||
|
||||
def __get_title(self, torrent: Any):
|
||||
# title default text
|
||||
if 'title' not in self.fields:
|
||||
return
|
||||
selector = self.fields.get('title', {})
|
||||
if 'selector' in selector:
|
||||
self.torrents_info['title'] = self._safe_query(torrent, selector)
|
||||
elif 'text' in selector:
|
||||
render_dict = {}
|
||||
if "title_default" in self.fields:
|
||||
title_default_selector = self.fields.get('title_default', {})
|
||||
title_default = self._safe_query(torrent, title_default_selector)
|
||||
render_dict.update({'title_default': title_default})
|
||||
if "title_optional" in self.fields:
|
||||
title_optional_selector = self.fields.get('title_optional', {})
|
||||
title_optional = self._safe_query(torrent, title_optional_selector)
|
||||
render_dict.update({'title_optional': title_optional})
|
||||
self.torrents_info['title'] = Template(selector.get('text')).render(fields=render_dict)
|
||||
self.torrents_info['title'] = self.__filter_text(self.torrents_info.get('title'),
|
||||
selector.get('filters'))
|
||||
|
||||
def __get_description(self, torrent: Any):
|
||||
# description text
|
||||
if 'description' not in self.fields:
|
||||
return
|
||||
selector = self.fields.get('description', {})
|
||||
if "selector" in selector or "selectors" in selector:
|
||||
# 对于selectors情况,需要特殊处理selector_config
|
||||
desc_selector = selector.copy()
|
||||
if "selectors" in selector and "selector" not in selector:
|
||||
desc_selector["selector"] = selector.get("selectors", "")
|
||||
self.torrents_info['description'] = self._safe_query(torrent, desc_selector)
|
||||
elif "text" in selector:
|
||||
render_dict = {}
|
||||
if "tags" in self.fields:
|
||||
tags_selector = self.fields.get('tags', {})
|
||||
tag = self._safe_query(torrent, tags_selector)
|
||||
render_dict.update({'tags': tag})
|
||||
if "subject" in self.fields:
|
||||
subject_selector = self.fields.get('subject', {})
|
||||
subject = self._safe_query(torrent, subject_selector)
|
||||
render_dict.update({'subject': subject})
|
||||
if "description_free_forever" in self.fields:
|
||||
description_free_forever_selector = self.fields.get("description_free_forever", {})
|
||||
description_free_forever = self._safe_query(torrent, description_free_forever_selector)
|
||||
render_dict.update({"description_free_forever": description_free_forever})
|
||||
if "description_normal" in self.fields:
|
||||
description_normal_selector = self.fields.get("description_normal", {})
|
||||
description_normal = self._safe_query(torrent, description_normal_selector)
|
||||
render_dict.update({"description_normal": description_normal})
|
||||
self.torrents_info['description'] = Template(selector.get('text')).render(fields=render_dict)
|
||||
self.torrents_info['description'] = self.__filter_text(self.torrents_info.get('description'),
|
||||
selector.get('filters'))
|
||||
|
||||
def __get_detail(self, torrent: Any):
|
||||
# details page text
|
||||
if 'details' not in self.fields:
|
||||
return
|
||||
selector = self.fields.get('details', {})
|
||||
item = self._safe_query(torrent, selector)
|
||||
detail_link = self.__filter_text(item, selector.get('filters'))
|
||||
if detail_link:
|
||||
if not detail_link.startswith("http"):
|
||||
if detail_link.startswith("//"):
|
||||
self.torrents_info['page_url'] = self.domain.split(":")[0] + ":" + detail_link
|
||||
elif detail_link.startswith("/"):
|
||||
self.torrents_info['page_url'] = self.domain + detail_link[1:]
|
||||
else:
|
||||
self.torrents_info['page_url'] = self.domain + detail_link
|
||||
else:
|
||||
self.torrents_info['page_url'] = detail_link
|
||||
|
||||
def __get_download(self, torrent: Any):
|
||||
# download link text
|
||||
if 'download' not in self.fields:
|
||||
return
|
||||
selector = self.fields.get('download', {})
|
||||
item = self._safe_query(torrent, selector)
|
||||
download_link = self.__filter_text(item, selector.get('filters'))
|
||||
if download_link:
|
||||
if not download_link.startswith("http") \
|
||||
and not download_link.startswith("magnet"):
|
||||
_scheme, _domain = StringUtils.get_url_netloc(self.domain)
|
||||
if _domain in download_link:
|
||||
if download_link.startswith("/"):
|
||||
self.torrents_info['enclosure'] = f"{_scheme}:{download_link}"
|
||||
else:
|
||||
self.torrents_info['enclosure'] = f"{_scheme}://{download_link}"
|
||||
else:
|
||||
if download_link.startswith("/"):
|
||||
self.torrents_info['enclosure'] = f"{self.domain}{download_link[1:]}"
|
||||
else:
|
||||
self.torrents_info['enclosure'] = f"{self.domain}{download_link}"
|
||||
else:
|
||||
self.torrents_info['enclosure'] = download_link
|
||||
|
||||
def __get_imdbid(self, torrent: Any):
|
||||
# imdbid
|
||||
if "imdbid" not in self.fields:
|
||||
return
|
||||
selector = self.fields.get('imdbid', {})
|
||||
item = self._safe_query(torrent, selector)
|
||||
self.torrents_info['imdbid'] = self.__filter_text(item, selector.get('filters'))
|
||||
|
||||
def __get_size(self, torrent: Any):
|
||||
# torrent size int
|
||||
if 'size' not in self.fields:
|
||||
return
|
||||
selector = self.fields.get('size', {})
|
||||
item = self._safe_query(torrent, selector)
|
||||
if item:
|
||||
size_val = item.replace("\n", "").strip()
|
||||
size_val = self.__filter_text(size_val,
|
||||
selector.get('filters'))
|
||||
self.torrents_info['size'] = StringUtils.num_filesize(size_val)
|
||||
else:
|
||||
self.torrents_info['size'] = 0
|
||||
|
||||
def __get_leechers(self, torrent: Any):
|
||||
# torrent leechers int
|
||||
if 'leechers' not in self.fields:
|
||||
return
|
||||
selector = self.fields.get('leechers', {})
|
||||
item = self._safe_query(torrent, selector)
|
||||
if item:
|
||||
peers_val = item.split("/")[0]
|
||||
peers_val = peers_val.replace(",", "")
|
||||
peers_val = self.__filter_text(peers_val, selector.get('filters'))
|
||||
self.torrents_info['peers'] = int(peers_val) if peers_val and peers_val.isdigit() else 0
|
||||
else:
|
||||
self.torrents_info['peers'] = 0
|
||||
|
||||
def __get_seeders(self, torrent: Any):
|
||||
# torrent seeders int
|
||||
if 'seeders' not in self.fields:
|
||||
return
|
||||
selector = self.fields.get('seeders', {})
|
||||
item = self._safe_query(torrent, selector)
|
||||
if item:
|
||||
seeders_val = item.split("/")[0]
|
||||
seeders_val = seeders_val.replace(",", "")
|
||||
seeders_val = self.__filter_text(seeders_val, selector.get('filters'))
|
||||
self.torrents_info['seeders'] = int(seeders_val) if seeders_val and seeders_val.isdigit() else 0
|
||||
else:
|
||||
self.torrents_info['seeders'] = 0
|
||||
|
||||
def __get_grabs(self, torrent: Any):
|
||||
# torrent grabs int
|
||||
if 'grabs' not in self.fields:
|
||||
return
|
||||
selector = self.fields.get('grabs', {})
|
||||
item = self._safe_query(torrent, selector)
|
||||
if item:
|
||||
grabs_val = item.split("/")[0]
|
||||
grabs_val = grabs_val.replace(",", "")
|
||||
grabs_val = self.__filter_text(grabs_val, selector.get('filters'))
|
||||
self.torrents_info['grabs'] = int(grabs_val) if grabs_val and grabs_val.isdigit() else 0
|
||||
else:
|
||||
self.torrents_info['grabs'] = 0
|
||||
|
||||
def __get_pubdate(self, torrent: Any):
|
||||
# torrent pubdate yyyy-mm-dd hh:mm:ss
|
||||
if 'date_added' not in self.fields:
|
||||
return
|
||||
selector = self.fields.get('date_added', {})
|
||||
pubdate_str = self._safe_query(torrent, selector)
|
||||
if pubdate_str:
|
||||
pubdate_str = pubdate_str.replace('\n', ' ').strip()
|
||||
self.torrents_info['pubdate'] = self.__filter_text(pubdate_str, selector.get('filters'))
|
||||
|
||||
def __get_date_elapsed(self, torrent: Any):
|
||||
# torrent date elapsed text
|
||||
if 'date_elapsed' not in self.fields:
|
||||
return
|
||||
selector = self.fields.get('date_elapsed', {})
|
||||
date_elapsed = self._safe_query(torrent, selector)
|
||||
self.torrents_info['date_elapsed'] = self.__filter_text(date_elapsed, selector.get('filters'))
|
||||
|
||||
def __get_downloadvolumefactor(self, torrent: Any):
|
||||
# downloadvolumefactor int
|
||||
selector = self.fields.get('downloadvolumefactor', {})
|
||||
if not selector:
|
||||
return
|
||||
self.torrents_info['downloadvolumefactor'] = 1
|
||||
if 'case' in selector:
|
||||
for downloadvolumefactorselector in list(selector.get('case', {}).keys()):
|
||||
downloadvolumefactor = torrent(downloadvolumefactorselector)
|
||||
try:
|
||||
if len(downloadvolumefactor) > 0:
|
||||
self.torrents_info['downloadvolumefactor'] = selector.get('case', {}).get(
|
||||
downloadvolumefactorselector)
|
||||
break
|
||||
finally:
|
||||
downloadvolumefactor.clear()
|
||||
del downloadvolumefactor
|
||||
elif "selector" in selector:
|
||||
item = self._safe_query(torrent, selector)
|
||||
if item:
|
||||
downloadvolumefactor = re.search(r'(\d+\.?\d*)', item)
|
||||
if downloadvolumefactor:
|
||||
self.torrents_info['downloadvolumefactor'] = int(downloadvolumefactor.group(1))
|
||||
|
||||
def __get_uploadvolumefactor(self, torrent: Any):
|
||||
# uploadvolumefactor int
|
||||
selector = self.fields.get('uploadvolumefactor', {})
|
||||
if not selector:
|
||||
return
|
||||
self.torrents_info['uploadvolumefactor'] = 1
|
||||
if 'case' in selector:
|
||||
for uploadvolumefactorselector in list(selector.get('case', {}).keys()):
|
||||
uploadvolumefactor = torrent(uploadvolumefactorselector)
|
||||
try:
|
||||
if len(uploadvolumefactor) > 0:
|
||||
self.torrents_info['uploadvolumefactor'] = selector.get('case', {}).get(
|
||||
uploadvolumefactorselector)
|
||||
break
|
||||
finally:
|
||||
uploadvolumefactor.clear()
|
||||
del uploadvolumefactor
|
||||
elif "selector" in selector:
|
||||
item = self._safe_query(torrent, selector)
|
||||
if item:
|
||||
uploadvolumefactor = re.search(r'(\d+\.?\d*)', item)
|
||||
if uploadvolumefactor:
|
||||
self.torrents_info['uploadvolumefactor'] = int(uploadvolumefactor.group(1))
|
||||
|
||||
def __get_labels(self, torrent: Any):
|
||||
# labels ['label1', 'label2']
|
||||
if 'labels' not in self.fields:
|
||||
return
|
||||
selector = self.fields.get('labels', {})
|
||||
if not selector.get('selector'):
|
||||
self.torrents_info['labels'] = []
|
||||
return
|
||||
|
||||
# labels需要特殊处理,因为它返回的是列表
|
||||
labels = torrent(selector.get("selector", "")).clone()
|
||||
try:
|
||||
self.__remove(labels, selector)
|
||||
items = self.__attribute_or_text(labels, selector)
|
||||
if items:
|
||||
self.torrents_info['labels'] = [item for item in items if item]
|
||||
else:
|
||||
self.torrents_info['labels'] = []
|
||||
finally:
|
||||
labels.clear()
|
||||
del labels
|
||||
|
||||
def __get_free_date(self, torrent: Any):
|
||||
# free date yyyy-mm-dd hh:mm:ss
|
||||
if 'freedate' not in self.fields:
|
||||
return
|
||||
selector = self.fields.get('freedate', {})
|
||||
freedate = self._safe_query(torrent, selector)
|
||||
self.torrents_info['freedate'] = self.__filter_text(freedate, selector.get('filters'))
|
||||
|
||||
def __get_hit_and_run(self, torrent: Any):
|
||||
# hitandrun True/False
|
||||
if 'hr' not in self.fields:
|
||||
return
|
||||
selector = self.fields.get('hr', {})
|
||||
hit_and_run = torrent(selector.get('selector', ''))
|
||||
try:
|
||||
if hit_and_run:
|
||||
self.torrents_info['hit_and_run'] = True
|
||||
else:
|
||||
self.torrents_info['hit_and_run'] = False
|
||||
finally:
|
||||
hit_and_run.clear()
|
||||
del hit_and_run
|
||||
|
||||
def __get_category(self, torrent: Any):
|
||||
# category 电影/电视剧
|
||||
if 'category' not in self.fields:
|
||||
return
|
||||
selector = self.fields.get('category', {})
|
||||
category_value = self._safe_query(torrent, selector)
|
||||
category_value = self.__filter_text(category_value, selector.get('filters'))
|
||||
if category_value and self.category:
|
||||
tv_cats = [str(cat.get("id")) for cat in self.category.get("tv") or []]
|
||||
movie_cats = [str(cat.get("id")) for cat in self.category.get("movie") or []]
|
||||
if category_value in tv_cats \
|
||||
and category_value not in movie_cats:
|
||||
self.torrents_info['category'] = MediaType.TV.value
|
||||
elif category_value in movie_cats:
|
||||
self.torrents_info['category'] = MediaType.MOVIE.value
|
||||
else:
|
||||
self.torrents_info['category'] = MediaType.UNKNOWN.value
|
||||
else:
|
||||
self.torrents_info['category'] = MediaType.UNKNOWN.value
|
||||
|
||||
def _safe_query(self, torrent: Any, selector_config: Optional[dict]) -> Optional[str]:
|
||||
"""
|
||||
安全地执行PyQuery查询并自动清理资源
|
||||
:param torrent: PyQuery对象
|
||||
:param selector_config: 选择器配置
|
||||
:return: 处理后的结果
|
||||
"""
|
||||
if not selector_config or not selector_config.get('selector'):
|
||||
return None
|
||||
|
||||
query_obj = torrent(selector_config.get('selector', '')).clone()
|
||||
try:
|
||||
self.__remove(query_obj, selector_config)
|
||||
items = self.__attribute_or_text(query_obj, selector_config)
|
||||
return self.__index(items, selector_config)
|
||||
finally:
|
||||
query_obj.clear()
|
||||
del query_obj
|
||||
|
||||
def get_info(self, torrent: Any) -> dict:
|
||||
"""
|
||||
解析单条种子数据
|
||||
"""
|
||||
# 每次调用时重新初始化,避免数据累积
|
||||
self.torrents_info = {}
|
||||
try:
|
||||
# 标题
|
||||
self.__get_title(torrent)
|
||||
# 描述
|
||||
self.__get_description(torrent)
|
||||
# 详情页面
|
||||
self.__get_detail(torrent)
|
||||
# 下载链接
|
||||
self.__get_download(torrent)
|
||||
# 完成数
|
||||
self.__get_grabs(torrent)
|
||||
# 下载数
|
||||
self.__get_leechers(torrent)
|
||||
# 做种数
|
||||
self.__get_seeders(torrent)
|
||||
# 大小
|
||||
self.__get_size(torrent)
|
||||
# IMDBID
|
||||
self.__get_imdbid(torrent)
|
||||
# 下载系数
|
||||
self.__get_downloadvolumefactor(torrent)
|
||||
# 上传系数
|
||||
self.__get_uploadvolumefactor(torrent)
|
||||
# 发布时间
|
||||
self.__get_pubdate(torrent)
|
||||
# 已发布时间
|
||||
self.__get_date_elapsed(torrent)
|
||||
# 免费载止时间
|
||||
self.__get_free_date(torrent)
|
||||
# 标签
|
||||
self.__get_labels(torrent)
|
||||
# HR
|
||||
self.__get_hit_and_run(torrent)
|
||||
# 分类
|
||||
self.__get_category(torrent)
|
||||
# 返回当前种子信息的副本,而不是引用
|
||||
return self.torrents_info.copy() if self.torrents_info else {}
|
||||
except Exception as err:
|
||||
logger.error("%s 搜索出现错误:%s" % (self.indexername, str(err)))
|
||||
return {}
|
||||
finally:
|
||||
self.torrents_info.clear()
|
||||
|
||||
@staticmethod
|
||||
def __filter_text(text: Optional[str], filters: Optional[List[dict]]) -> str:
|
||||
"""
|
||||
对文件进行处理
|
||||
"""
|
||||
if not text or not filters or not isinstance(filters, list):
|
||||
return text
|
||||
if not isinstance(text, str):
|
||||
text = str(text)
|
||||
for filter_item in filters:
|
||||
if not text:
|
||||
break
|
||||
method_name = filter_item.get("name")
|
||||
try:
|
||||
args = filter_item.get("args")
|
||||
if method_name == "re_search" and isinstance(args, list):
|
||||
rematch = re.search(r"%s" % args[0], text)
|
||||
if rematch:
|
||||
text = rematch.group(args[-1])
|
||||
elif method_name == "split" and isinstance(args, list):
|
||||
text = text.split(r"%s" % args[0])[args[-1]]
|
||||
elif method_name == "replace" and isinstance(args, list):
|
||||
text = text.replace(r"%s" % args[0], r"%s" % args[-1])
|
||||
elif method_name == "dateparse" and isinstance(args, str):
|
||||
text = text.replace("\n", " ").strip()
|
||||
text = datetime.datetime.strptime(text, r"%s" % args)
|
||||
elif method_name == "strip":
|
||||
text = text.strip()
|
||||
elif method_name == "appendleft":
|
||||
text = f"{args}{text}"
|
||||
elif method_name == "querystring":
|
||||
parsed_url = urlparse(str(text))
|
||||
query_params = parse_qs(parsed_url.query)
|
||||
param_value = query_params.get(args)
|
||||
text = param_value[0] if param_value else ''
|
||||
except Exception as err:
|
||||
logger.debug(f'过滤器 {method_name} 处理失败:{str(err)} - {traceback.format_exc()}')
|
||||
return text.strip()
|
||||
|
||||
@staticmethod
|
||||
def __remove(item: Any, selector: Optional[dict]):
|
||||
"""
|
||||
移除元素
|
||||
"""
|
||||
if selector and "remove" in selector:
|
||||
removelist = selector.get('remove', '').split(', ')
|
||||
for v in removelist:
|
||||
item.remove(v)
|
||||
|
||||
@staticmethod
|
||||
def __attribute_or_text(item: Any, selector: Optional[dict]) -> list:
|
||||
if not selector:
|
||||
return item
|
||||
if not item:
|
||||
return []
|
||||
if 'attribute' in selector:
|
||||
items = [i.attr(selector.get('attribute')) for i in item.items() if i]
|
||||
else:
|
||||
items = [i.text() for i in item.items() if i]
|
||||
return items
|
||||
|
||||
@staticmethod
|
||||
def __index(items: Optional[list], selector: Optional[dict]) -> Optional[str]:
|
||||
if not items:
|
||||
return None
|
||||
if selector:
|
||||
if "contents" in selector \
|
||||
and len(items) > int(selector.get("contents")):
|
||||
item = items[0].split("\n")[selector.get("contents")]
|
||||
elif "index" in selector \
|
||||
and len(items) > int(selector.get("index")):
|
||||
item = items[int(selector.get("index"))]
|
||||
else:
|
||||
item = items[0]
|
||||
else:
|
||||
item = items[0]
|
||||
return item
|
||||
|
||||
def parse(self, html_text: str) -> List[dict]:
|
||||
"""
|
||||
解析整个页面
|
||||
"""
|
||||
if not html_text:
|
||||
self.is_error = True
|
||||
return []
|
||||
|
||||
# 清空旧结果
|
||||
self.torrents_info_array = []
|
||||
html_doc = None
|
||||
try:
|
||||
# 解析站点文本对象
|
||||
html_doc = PyQuery(html_text)
|
||||
# 种子筛选器
|
||||
torrents_selector = self.list.get('selector', '')
|
||||
# 遍历种子html列表
|
||||
for i, torn in enumerate(html_doc(torrents_selector)):
|
||||
if i >= int(self.result_num):
|
||||
break
|
||||
# 创建临时PyQuery对象进行解析
|
||||
torrent_query = PyQuery(torn)
|
||||
try:
|
||||
# 直接获取种子信息,避免深拷贝
|
||||
torrent_info = self.get_info(torrent_query)
|
||||
if torrent_info:
|
||||
# 浅拷贝即可,减少内存使用
|
||||
self.torrents_info_array.append(torrent_info)
|
||||
finally:
|
||||
# 显式删除临时PyQuery对象
|
||||
torrent_query.clear()
|
||||
del torrent_query
|
||||
# 返回数组的副本,防止被后续清理操作影响
|
||||
return self.torrents_info_array.copy()
|
||||
except Exception as err:
|
||||
self.is_error = True
|
||||
logger.warn(f"错误:{self.indexername} {str(err)}")
|
||||
return []
|
||||
finally:
|
||||
# 清理种子缓存
|
||||
self.torrents_info_array.clear()
|
||||
# 清理HTML文档对象
|
||||
if html_doc is not None:
|
||||
html_doc.clear()
|
||||
del html_doc
|
||||
# 清理html_text引用
|
||||
del html_text
|
||||
|
||||
@@ -168,7 +168,8 @@ class Jellyfin:
|
||||
path=library.get("Path"),
|
||||
type=library_type,
|
||||
image=image,
|
||||
link=link
|
||||
link=link,
|
||||
server_type="jellyfin"
|
||||
))
|
||||
return libraries
|
||||
|
||||
@@ -934,7 +935,8 @@ class Jellyfin:
|
||||
type=item_type,
|
||||
image=image,
|
||||
link=link,
|
||||
percent=item.get("UserData", {}).get("PlayedPercentage")
|
||||
percent=item.get("UserData", {}).get("PlayedPercentage"),
|
||||
server_type='jellyfin',
|
||||
))
|
||||
return ret_resume
|
||||
else:
|
||||
@@ -986,7 +988,8 @@ class Jellyfin:
|
||||
type=item_type,
|
||||
image=image,
|
||||
link=link,
|
||||
BackdropImageTags=item.get("BackdropImageTags")
|
||||
BackdropImageTags=item.get("BackdropImageTags"),
|
||||
server_type='jellyfin'
|
||||
))
|
||||
return ret_latest
|
||||
else:
|
||||
|
||||
@@ -154,7 +154,8 @@ class Plex:
|
||||
type=library_type,
|
||||
image_list=image_list,
|
||||
link=f"{self._playhost or self._host}web/index.html#!/media/{self._plex.machineIdentifier}"
|
||||
f"/com.plexapp.plugins.library?source={library.key}"
|
||||
f"/com.plexapp.plugins.library?source={library.key}&X-Plex-Token={self._token}",
|
||||
server_type='plex'
|
||||
)
|
||||
)
|
||||
return libraries
|
||||
@@ -378,7 +379,7 @@ class Plex:
|
||||
file_path = item.target_path
|
||||
lib_key, path = self.__find_librarie(file_path, self._libraries)
|
||||
# 如果存在同一剧集的多集,key(path)相同会合并
|
||||
result_dict[path] = lib_key
|
||||
result_dict[path.as_posix()] = lib_key
|
||||
if "" in result_dict:
|
||||
# 如果有匹配失败的,刷新整个库
|
||||
self._plex.library.update()
|
||||
@@ -386,10 +387,12 @@ class Plex:
|
||||
# 否则一个一个刷新
|
||||
for path, lib_key in result_dict.items():
|
||||
logger.info(f"刷新媒体库:{lib_key} - {path}")
|
||||
self._plex.query(f'/library/sections/{lib_key}/refresh?path={quote_plus(str(Path(path).parent))}')
|
||||
self._plex.query(f'/library/sections/{lib_key}/refresh?path={quote_plus(Path(path).parent.as_posix())}')
|
||||
return None
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def __find_librarie(path: Path, libraries: List[Any]) -> Tuple[str, str]:
|
||||
def __find_librarie(path: Path, libraries: List[Any]) -> Tuple[str, Optional[Path]]:
|
||||
"""
|
||||
判断这个path属于哪个媒体库
|
||||
多个媒体库配置的目录不应有重复和嵌套,
|
||||
@@ -404,17 +407,17 @@ class Plex:
|
||||
return _path.parts[:len(_parent.parts)] == _parent.parts
|
||||
|
||||
if path is None:
|
||||
return "", ""
|
||||
return "", None
|
||||
|
||||
try:
|
||||
for lib in libraries:
|
||||
if hasattr(lib, "locations") and lib.locations:
|
||||
for location in lib.locations:
|
||||
if is_subpath(path, Path(location)):
|
||||
return lib.key, str(path)
|
||||
return lib.key, path
|
||||
except Exception as err:
|
||||
logger.error(f"查找媒体库出错:{str(err)}")
|
||||
return "", ""
|
||||
return "", None
|
||||
|
||||
def get_iteminfo(self, itemid: str) -> Optional[schemas.MediaServerItem]:
|
||||
"""
|
||||
@@ -541,6 +544,7 @@ class Plex:
|
||||
continue
|
||||
except Exception as err:
|
||||
logger.error(f"获取媒体库列表出错:{str(err)}")
|
||||
return None
|
||||
|
||||
def get_webhook_message(self, form: any) -> Optional[schemas.WebhookEventInfo]:
|
||||
"""
|
||||
@@ -718,7 +722,7 @@ class Plex:
|
||||
拼装媒体播放链接
|
||||
:param item_id: 媒体的的ID
|
||||
"""
|
||||
return f'{self._playhost or self._host}web/index.html#!/server/{self._plex.machineIdentifier}/details?key={item_id}'
|
||||
return f'{self._playhost or self._host}web/index.html#!/server/{self._plex.machineIdentifier}/details?key={item_id}&X-Plex-Token={self._token}'
|
||||
|
||||
def get_resume(self, num: Optional[int] = 12) -> Optional[List[schemas.MediaServerPlayItem]]:
|
||||
"""
|
||||
@@ -752,7 +756,8 @@ class Plex:
|
||||
type=item_type,
|
||||
image=image,
|
||||
link=link,
|
||||
percent=item.viewOffset / item.duration * 100 if item.viewOffset and item.duration else 0
|
||||
percent=item.viewOffset / item.duration * 100 if item.viewOffset and item.duration else 0,
|
||||
server_type='plex'
|
||||
))
|
||||
return ret_resume[:num]
|
||||
|
||||
@@ -820,7 +825,8 @@ class Plex:
|
||||
subtitle=item.year,
|
||||
type=item_type,
|
||||
image=image,
|
||||
link=link
|
||||
link=link,
|
||||
server_type='plex'
|
||||
))
|
||||
offset += num
|
||||
return ret_resume[:num]
|
||||
|
||||
@@ -104,20 +104,24 @@ class SubtitleModule(_ModuleBase):
|
||||
logger.warn(f"读取页面代码失败:{torrent.page_url}")
|
||||
return
|
||||
html = etree.HTML(res.text)
|
||||
sublink_list = []
|
||||
for xpath in self._SITE_SUBTITLE_XPATH:
|
||||
sublinks = html.xpath(xpath)
|
||||
if sublinks:
|
||||
for sublink in sublinks:
|
||||
if not sublink:
|
||||
continue
|
||||
if not sublink.startswith("http"):
|
||||
base_url = StringUtils.get_base_url(torrent.page_url)
|
||||
if sublink.startswith("/"):
|
||||
sublink = "%s%s" % (base_url, sublink)
|
||||
else:
|
||||
sublink = "%s/%s" % (base_url, sublink)
|
||||
sublink_list.append(sublink)
|
||||
try:
|
||||
sublink_list = []
|
||||
for xpath in self._SITE_SUBTITLE_XPATH:
|
||||
sublinks = html.xpath(xpath)
|
||||
if sublinks:
|
||||
for sublink in sublinks:
|
||||
if not sublink:
|
||||
continue
|
||||
if not sublink.startswith("http"):
|
||||
base_url = StringUtils.get_base_url(torrent.page_url)
|
||||
if sublink.startswith("/"):
|
||||
sublink = "%s%s" % (base_url, sublink)
|
||||
else:
|
||||
sublink = "%s/%s" % (base_url, sublink)
|
||||
sublink_list.append(sublink)
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
# 下载所有字幕文件
|
||||
for sublink in sublink_list:
|
||||
logger.info(f"找到字幕下载链接:{sublink},开始下载...")
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
from typing import Dict
|
||||
from typing import Optional, Union, List, Tuple, Any
|
||||
|
||||
@@ -187,28 +188,33 @@ class TelegramModule(_ModuleBase, _MessageBase[Telegram]):
|
||||
)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _handle_text_message(msg: dict, client_config: NotificationConf, client: Telegram) -> Optional[CommingMessage]:
|
||||
def _handle_text_message(self, msg: dict,
|
||||
client_config: NotificationConf, client: Telegram) -> Optional[CommingMessage]:
|
||||
"""
|
||||
处理普通文本消息
|
||||
"""
|
||||
text = msg.get("text")
|
||||
user_id = msg.get("from", {}).get("id")
|
||||
user_name = msg.get("from", {}).get("username")
|
||||
# Extract chat_id to enable correct reply targeting
|
||||
chat_id = msg.get("chat", {}).get("id")
|
||||
|
||||
if text and user_id:
|
||||
logger.info(f"收到来自 {client_config.name} 的Telegram消息:"
|
||||
f"userid={user_id}, username={user_name}, text={text}")
|
||||
f"userid={user_id}, username={user_name}, chat_id={chat_id}, text={text}")
|
||||
|
||||
# Clean bot mentions from text to ensure consistent processing
|
||||
cleaned_text = self._clean_bot_mention(text, client.bot_username if client else None)
|
||||
|
||||
# 检查权限
|
||||
admin_users = client_config.config.get("TELEGRAM_ADMINS")
|
||||
user_list = client_config.config.get("TELEGRAM_USERS")
|
||||
chat_id = client_config.config.get("TELEGRAM_CHAT_ID")
|
||||
config_chat_id = client_config.config.get("TELEGRAM_CHAT_ID")
|
||||
|
||||
if text.startswith("/"):
|
||||
if cleaned_text.startswith("/"):
|
||||
if admin_users \
|
||||
and str(user_id) not in admin_users.split(',') \
|
||||
and str(user_id) != chat_id:
|
||||
and str(user_id) != config_chat_id:
|
||||
client.send_msg(title="只有管理员才有权限执行此命令", userid=user_id)
|
||||
return None
|
||||
else:
|
||||
@@ -223,10 +229,38 @@ class TelegramModule(_ModuleBase, _MessageBase[Telegram]):
|
||||
source=client_config.name,
|
||||
userid=user_id,
|
||||
username=user_name,
|
||||
text=text
|
||||
text=cleaned_text, # Use cleaned text
|
||||
chat_id=str(chat_id) if chat_id else None
|
||||
)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _clean_bot_mention(text: str, bot_username: Optional[str]) -> str:
|
||||
"""
|
||||
清理消息中的@bot部分,确保文本处理一致性
|
||||
:param text: 原始消息文本
|
||||
:param bot_username: bot用户名
|
||||
:return: 清理后的文本
|
||||
"""
|
||||
if not text or not bot_username:
|
||||
return text
|
||||
|
||||
# Remove @bot_username from the beginning and any position in text
|
||||
cleaned = text
|
||||
mention_pattern = f"@{bot_username}"
|
||||
|
||||
# Remove mention at the beginning with optional following space
|
||||
if cleaned.startswith(mention_pattern):
|
||||
cleaned = cleaned[len(mention_pattern):].lstrip()
|
||||
|
||||
# Remove mention at any other position
|
||||
cleaned = cleaned.replace(mention_pattern, "").strip()
|
||||
|
||||
# Clean up multiple spaces
|
||||
cleaned = re.sub(r'\s+', ' ', cleaned).strip()
|
||||
|
||||
return cleaned
|
||||
|
||||
def post_message(self, message: Notification) -> None:
|
||||
"""
|
||||
发送消息
|
||||
|
||||
@@ -25,6 +25,8 @@ class Telegram:
|
||||
_event = Event()
|
||||
_bot: telebot.TeleBot = None
|
||||
_callback_handlers: Dict[str, Callable] = {} # 存储回调处理器
|
||||
_user_chat_mapping: Dict[str, str] = {} # userid -> chat_id mapping for reply targeting
|
||||
_bot_username: Optional[str] = None # Bot username for mention detection
|
||||
|
||||
def __init__(self, TELEGRAM_TOKEN: Optional[str] = None, TELEGRAM_CHAT_ID: Optional[str] = None, **kwargs):
|
||||
"""
|
||||
@@ -49,6 +51,15 @@ class Telegram:
|
||||
_bot = telebot.TeleBot(self._telegram_token, parse_mode="Markdown")
|
||||
# 记录句柄
|
||||
self._bot = _bot
|
||||
# 获取并存储bot用户名用于@检测
|
||||
try:
|
||||
bot_info = _bot.get_me()
|
||||
self._bot_username = bot_info.username
|
||||
logger.info(f"Telegram bot用户名: @{self._bot_username}")
|
||||
except Exception as e:
|
||||
logger.error(f"获取bot信息失败: {e}")
|
||||
self._bot_username = None
|
||||
|
||||
# 标记渠道来源
|
||||
if kwargs.get("name"):
|
||||
self._ds_url = f"{self._ds_url}&source={kwargs.get('name')}"
|
||||
@@ -59,7 +70,12 @@ class Telegram:
|
||||
|
||||
@_bot.message_handler(func=lambda message: True)
|
||||
def echo_all(message):
|
||||
RequestUtils(timeout=15).post_res(self._ds_url, json=message.json)
|
||||
# Update user-chat mapping when receiving messages
|
||||
self._update_user_chat_mapping(message.from_user.id, message.chat.id)
|
||||
|
||||
# Check if we should process this message
|
||||
if self._should_process_message(message):
|
||||
RequestUtils(timeout=15).post_res(self._ds_url, json=message.json)
|
||||
|
||||
@_bot.callback_query_handler(func=lambda call: True)
|
||||
def callback_query(call):
|
||||
@@ -67,6 +83,9 @@ class Telegram:
|
||||
处理按钮点击回调
|
||||
"""
|
||||
try:
|
||||
# Update user-chat mapping for callbacks too
|
||||
self._update_user_chat_mapping(call.from_user.id, call.message.chat.id)
|
||||
|
||||
# 解析回调数据
|
||||
callback_data = call.data
|
||||
user_id = str(call.from_user.id)
|
||||
@@ -94,8 +113,8 @@ class Telegram:
|
||||
# 发送给主程序处理
|
||||
RequestUtils(timeout=15).post_res(self._ds_url, json=callback_json)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理按钮回调失败:{str(e)}")
|
||||
except Exception as err:
|
||||
logger.error(f"处理按钮回调失败:{str(err)}")
|
||||
_bot.answer_callback_query(call.id, "处理失败,请重试")
|
||||
|
||||
def run_polling():
|
||||
@@ -112,6 +131,76 @@ class Telegram:
|
||||
self._polling_thread.start()
|
||||
logger.info("Telegram消息接收服务启动")
|
||||
|
||||
@property
|
||||
def bot_username(self) -> Optional[str]:
|
||||
"""
|
||||
获取Bot用户名
|
||||
:return: Bot用户名或None
|
||||
"""
|
||||
return self._bot_username
|
||||
|
||||
def _update_user_chat_mapping(self, userid: int, chat_id: int) -> None:
|
||||
"""
|
||||
更新用户与聊天的映射关系
|
||||
:param userid: 用户ID
|
||||
:param chat_id: 聊天ID
|
||||
"""
|
||||
if userid and chat_id:
|
||||
self._user_chat_mapping[str(userid)] = str(chat_id)
|
||||
|
||||
def _get_user_chat_id(self, userid: str) -> Optional[str]:
|
||||
"""
|
||||
获取用户对应的聊天ID
|
||||
:param userid: 用户ID
|
||||
:return: 聊天ID或None
|
||||
"""
|
||||
return self._user_chat_mapping.get(str(userid)) if userid else None
|
||||
|
||||
def _should_process_message(self, message) -> bool:
|
||||
"""
|
||||
判断是否应该处理这条消息
|
||||
:param message: Telegram消息对象
|
||||
:return: 是否处理
|
||||
"""
|
||||
# 私聊消息总是处理
|
||||
if message.chat.type == 'private':
|
||||
logger.debug(f"处理私聊消息:用户 {message.from_user.id}")
|
||||
return True
|
||||
|
||||
# 群聊中的命令消息总是处理(以/开头)
|
||||
if message.text and message.text.startswith('/'):
|
||||
logger.debug(f"处理群聊命令消息:{message.text[:20]}...")
|
||||
return True
|
||||
|
||||
# 群聊中检查是否@了机器人
|
||||
if message.chat.type in ['group', 'supergroup']:
|
||||
if not self._bot_username:
|
||||
# 如果没有获取到bot用户名,为了安全起见处理所有消息
|
||||
logger.debug("未获取到bot用户名,处理所有群聊消息")
|
||||
return True
|
||||
|
||||
# 检查消息文本中是否包含@bot_username
|
||||
if message.text and f"@{self._bot_username}" in message.text:
|
||||
logger.debug(f"检测到@{self._bot_username},处理群聊消息")
|
||||
return True
|
||||
|
||||
# 检查消息实体中是否有提及bot
|
||||
if message.entities:
|
||||
for entity in message.entities:
|
||||
if entity.type == 'mention':
|
||||
mention_text = message.text[entity.offset:entity.offset + entity.length]
|
||||
if mention_text == f"@{self._bot_username}":
|
||||
logger.debug(f"通过实体检测到@{self._bot_username},处理群聊消息")
|
||||
return True
|
||||
|
||||
# 群聊中没有@机器人,不处理
|
||||
logger.debug(f"群聊消息未@机器人,跳过处理:{message.text[:30] if message.text else 'No text'}...")
|
||||
return False
|
||||
|
||||
# 其他类型的聊天默认处理
|
||||
logger.debug(f"处理其他类型聊天消息:{message.chat.type}")
|
||||
return True
|
||||
|
||||
def get_state(self) -> bool:
|
||||
"""
|
||||
获取状态
|
||||
@@ -153,10 +242,8 @@ class Telegram:
|
||||
if link:
|
||||
caption = f"{caption}\n[查看详情]({link})"
|
||||
|
||||
if userid:
|
||||
chat_id = userid
|
||||
else:
|
||||
chat_id = self._telegram_chat_id
|
||||
# Determine target chat_id with improved logic using user mapping
|
||||
chat_id = self._determine_target_chat_id(userid, original_chat_id)
|
||||
|
||||
# 创建按钮键盘
|
||||
reply_markup = None
|
||||
@@ -175,6 +262,29 @@ class Telegram:
|
||||
logger.error(f"发送消息失败:{msg_e}")
|
||||
return False
|
||||
|
||||
def _determine_target_chat_id(self, userid: Optional[str] = None,
|
||||
original_chat_id: Optional[str] = None) -> str:
|
||||
"""
|
||||
确定目标聊天ID,使用用户映射确保回复到正确的聊天
|
||||
:param userid: 用户ID
|
||||
:param original_chat_id: 原消息的聊天ID
|
||||
:return: 目标聊天ID
|
||||
"""
|
||||
# 1. 优先使用原消息的聊天ID (编辑消息场景)
|
||||
if original_chat_id:
|
||||
return original_chat_id
|
||||
|
||||
# 2. 如果有userid,尝试从映射中获取用户的聊天ID
|
||||
if userid:
|
||||
mapped_chat_id = self._get_user_chat_id(userid)
|
||||
if mapped_chat_id:
|
||||
return mapped_chat_id
|
||||
# 如果映射中没有,回退到使用userid作为聊天ID (私聊场景)
|
||||
return userid
|
||||
|
||||
# 3. 最后使用默认聊天ID
|
||||
return self._telegram_chat_id
|
||||
|
||||
def send_medias_msg(self, medias: List[MediaInfo], userid: Optional[str] = None,
|
||||
title: Optional[str] = None, link: Optional[str] = None,
|
||||
buttons: Optional[List[List[Dict]]] = None,
|
||||
@@ -216,10 +326,8 @@ class Telegram:
|
||||
if link:
|
||||
caption = f"{caption}\n[查看详情]({link})"
|
||||
|
||||
if userid:
|
||||
chat_id = userid
|
||||
else:
|
||||
chat_id = self._telegram_chat_id
|
||||
# Determine target chat_id with improved logic using user mapping
|
||||
chat_id = self._determine_target_chat_id(userid, original_chat_id)
|
||||
|
||||
# 创建按钮键盘
|
||||
reply_markup = None
|
||||
@@ -278,10 +386,8 @@ class Telegram:
|
||||
if link:
|
||||
caption = f"{caption}\n[查看详情]({link})"
|
||||
|
||||
if userid:
|
||||
chat_id = userid
|
||||
else:
|
||||
chat_id = self._telegram_chat_id
|
||||
# Determine target chat_id with improved logic using user mapping
|
||||
chat_id = self._determine_target_chat_id(userid, original_chat_id)
|
||||
|
||||
# 创建按钮键盘
|
||||
reply_markup = None
|
||||
@@ -418,7 +524,7 @@ class Telegram:
|
||||
:param reply_markup: 内联键盘
|
||||
"""
|
||||
if image:
|
||||
res = RequestUtils(proxies=settings.PROXY, ua=settings.USER_AGENT).get_res(image)
|
||||
res = RequestUtils(proxies=settings.PROXY, ua=settings.NORMAL_USER_AGENT).get_res(image)
|
||||
if res is None:
|
||||
raise Exception("获取图片失败")
|
||||
if res.content:
|
||||
|
||||
@@ -7,19 +7,19 @@ from ruamel.yaml import CommentedMap
|
||||
|
||||
from app.core.config import settings
|
||||
from app.log import logger
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.singleton import WeakSingleton
|
||||
|
||||
|
||||
class CategoryHelper(metaclass=Singleton):
|
||||
class CategoryHelper(metaclass=WeakSingleton):
|
||||
"""
|
||||
二级分类
|
||||
"""
|
||||
_categorys = {}
|
||||
_movie_categorys = {}
|
||||
_tv_categorys = {}
|
||||
|
||||
def __init__(self):
|
||||
self._category_path: Path = settings.CONFIG_PATH / "category.yaml"
|
||||
self._categorys = {}
|
||||
self._movie_categorys = {}
|
||||
self._tv_categorys = {}
|
||||
self.init()
|
||||
|
||||
def init(self):
|
||||
@@ -69,7 +69,7 @@ class CategoryHelper(metaclass=Singleton):
|
||||
"""
|
||||
if not self._movie_categorys:
|
||||
return []
|
||||
return self._movie_categorys.keys()
|
||||
return list(self._movie_categorys.keys())
|
||||
|
||||
@property
|
||||
def tv_categorys(self) -> list:
|
||||
@@ -78,7 +78,7 @@ class CategoryHelper(metaclass=Singleton):
|
||||
"""
|
||||
if not self._tv_categorys:
|
||||
return []
|
||||
return self._tv_categorys.keys()
|
||||
return list(self._tv_categorys.keys())
|
||||
|
||||
def get_movie_category(self, tmdb_info) -> str:
|
||||
"""
|
||||
@@ -127,7 +127,7 @@ class CategoryHelper(metaclass=Singleton):
|
||||
continue
|
||||
elif attr == "production_countries":
|
||||
# 制片国家
|
||||
info_values = [str(val.get("iso_3166_1")).upper() for val in info_value]
|
||||
info_values = [str(val.get("iso_3166_1")).upper() for val in info_value] # type: ignore
|
||||
else:
|
||||
if isinstance(info_value, list):
|
||||
info_values = [str(val).upper() for val in info_value]
|
||||
|
||||
@@ -9,7 +9,7 @@ from typing import Optional
|
||||
from app.core.config import settings
|
||||
from app.core.meta import MetaBase
|
||||
from app.log import logger
|
||||
from app.utils.singleton import Singleton
|
||||
from app.utils.singleton import WeakSingleton
|
||||
from app.schemas.types import MediaType
|
||||
|
||||
lock = RLock()
|
||||
@@ -18,7 +18,7 @@ CACHE_EXPIRE_TIMESTAMP_STR = "cache_expire_timestamp"
|
||||
EXPIRE_TIMESTAMP = settings.CONF.meta
|
||||
|
||||
|
||||
class TmdbCache(metaclass=Singleton):
|
||||
class TmdbCache(metaclass=WeakSingleton):
|
||||
"""
|
||||
TMDB缓存数据
|
||||
{
|
||||
@@ -28,9 +28,6 @@ class TmdbCache(metaclass=Singleton):
|
||||
"type": MediaType
|
||||
}
|
||||
"""
|
||||
_meta_data: dict = {}
|
||||
# 缓存文件路径
|
||||
_meta_path: Path = None
|
||||
# TMDB缓存过期
|
||||
_tmdb_cache_expire: bool = True
|
||||
|
||||
@@ -218,3 +215,6 @@ class TmdbCache(metaclass=Singleton):
|
||||
if not cache_media_info:
|
||||
return
|
||||
self._meta_data[key]['title'] = cn_title
|
||||
|
||||
def __del__(self):
|
||||
self.save()
|
||||
|
||||
@@ -514,7 +514,7 @@ class TmdbApi:
|
||||
return {}
|
||||
logger.info("正在从TheDbMovie网站查询:%s ..." % name)
|
||||
tmdb_url = "https://www.themoviedb.org/search?query=%s" % quote(name)
|
||||
res = RequestUtils(timeout=5, ua=settings.USER_AGENT, proxies=settings.PROXY).get_res(url=tmdb_url)
|
||||
res = RequestUtils(timeout=5, ua=settings.NORMAL_USER_AGENT, proxies=settings.PROXY).get_res(url=tmdb_url)
|
||||
if res is None:
|
||||
return None
|
||||
if res.status_code == 429:
|
||||
@@ -563,6 +563,9 @@ class TmdbApi:
|
||||
except Exception as err:
|
||||
logger.error(f"从TheDbMovie网站查询出错:{str(err)}")
|
||||
return {}
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
return {}
|
||||
|
||||
def get_info(self,
|
||||
|
||||
@@ -25,7 +25,7 @@ class Transmission:
|
||||
若不设置参数,则创建配置文件设置的下载器
|
||||
"""
|
||||
if host and port:
|
||||
self._protocol, self._host, self._port = kwargs.get("protocol", self._protocol), host, port
|
||||
self._protocol, self._host, self._port = kwargs.get("protocol", "http"), host, port
|
||||
elif host:
|
||||
result = UrlUtils.parse_url_params(url=host)
|
||||
if result:
|
||||
|
||||
@@ -111,7 +111,7 @@ class Api:
|
||||
"_api_path",
|
||||
"_request_utils",
|
||||
"_version",
|
||||
"_session"
|
||||
"_session",
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -287,6 +287,18 @@ class Api:
|
||||
return True
|
||||
return False
|
||||
|
||||
def task_running(self):
|
||||
"""
|
||||
当前正在运行的任务
|
||||
"""
|
||||
if (
|
||||
res := self.__request_api("/task/running")
|
||||
) and res.success:
|
||||
if res.data:
|
||||
# TODO 具体正在运行的任务
|
||||
return True
|
||||
return False
|
||||
|
||||
def __build_item(self, info: dict) -> Item:
|
||||
"""
|
||||
构造媒体Item
|
||||
@@ -308,7 +320,7 @@ class Api:
|
||||
types=None,
|
||||
exclude_grouped_video=True,
|
||||
page=1,
|
||||
page_size=22,
|
||||
page_size=20,
|
||||
sort_by="create_time",
|
||||
sort="DESC",
|
||||
) -> Optional[list[Item]]:
|
||||
|
||||
@@ -111,6 +111,8 @@ class TrimeMedia:
|
||||
if self._userinfo is None:
|
||||
return False
|
||||
logger.debug(f"{self._username} 成功登录飞牛影视")
|
||||
# 刷新媒体库列表
|
||||
self.get_librarys()
|
||||
return True
|
||||
|
||||
def disconnect(self):
|
||||
@@ -161,6 +163,7 @@ class TrimeMedia:
|
||||
for img_path in library.posters or []
|
||||
],
|
||||
link=f"{self._playhost or self._api.host}/library/{library.guid}",
|
||||
server_type='trimemedia'
|
||||
)
|
||||
)
|
||||
return libraries
|
||||
@@ -311,6 +314,8 @@ class TrimeMedia:
|
||||
logger.error("飞牛仅支持管理员账号刷新媒体库")
|
||||
return False
|
||||
|
||||
# 必须调用 否则容易误报 -14 Task duplicate
|
||||
self._api.task_running()
|
||||
logger.info("刷新所有媒体库")
|
||||
return self._api.mdb_scanall()
|
||||
|
||||
@@ -337,6 +342,8 @@ class TrimeMedia:
|
||||
# 媒体库去重
|
||||
libraries.add(lib.guid)
|
||||
|
||||
# 必须调用 否则容易误报 -14 Task duplicate
|
||||
self._api.task_running()
|
||||
for lib_guid in libraries:
|
||||
# 逐个刷新
|
||||
lib = self._libraries[lib_guid]
|
||||
@@ -452,6 +459,7 @@ class TrimeMedia:
|
||||
if item.duration and item.ts is not None
|
||||
else 0
|
||||
),
|
||||
server_type='trimemedia',
|
||||
)
|
||||
|
||||
def get_items(
|
||||
|
||||
211
app/monitor.py
211
app/monitor.py
@@ -59,26 +59,21 @@ class Monitor(metaclass=Singleton):
|
||||
目录监控处理链,单例模式
|
||||
"""
|
||||
|
||||
# 退出事件
|
||||
_event = threading.Event()
|
||||
|
||||
# 监控服务
|
||||
_observers = []
|
||||
|
||||
# 定时服务
|
||||
_scheduler = None
|
||||
|
||||
# 存储快照缓存目录
|
||||
_snapshot_cache_dir = None
|
||||
|
||||
# 存储过照间隔(分钟)
|
||||
_snapshot_interval = 5
|
||||
|
||||
# TTL缓存,10秒钟有效
|
||||
_cache = TTLCache(maxsize=1024, ttl=10)
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
# 退出事件
|
||||
self._event = threading.Event()
|
||||
# 监控服务
|
||||
self._observers = []
|
||||
# 定时服务
|
||||
self._scheduler = None
|
||||
# 存储快照缓存目录
|
||||
self._snapshot_cache_dir = None
|
||||
# 存储过照间隔(分钟)
|
||||
self._snapshot_interval = 5
|
||||
# TTL缓存,10秒钟有效
|
||||
self._cache = TTLCache(maxsize=1024, ttl=10)
|
||||
# 监控的文件扩展名
|
||||
self.all_exts = settings.RMT_MEDIAEXT
|
||||
# 初始化快照缓存目录
|
||||
self._snapshot_cache_dir = settings.TEMP_PATH / "snapshots"
|
||||
@@ -100,17 +95,22 @@ class Monitor(metaclass=Singleton):
|
||||
logger.info("配置变更事件触发,重新初始化目录监控...")
|
||||
self.init()
|
||||
|
||||
def save_snapshot(self, storage: str, snapshot: Dict, file_count: int = 0):
|
||||
def save_snapshot(self, storage: str, snapshot: Dict, file_count: int = 0,
|
||||
last_snapshot_time: Optional[float] = None):
|
||||
"""
|
||||
保存快照到文件
|
||||
:param storage: 存储名称
|
||||
:param snapshot: 快照数据
|
||||
:param last_snapshot_time: 上次快照时间戳
|
||||
:param file_count: 文件数量,用于调整监控间隔
|
||||
"""
|
||||
try:
|
||||
cache_file = self._snapshot_cache_dir / f"{storage}_snapshot.json"
|
||||
snapshot_time = max((item.get('modify_time', 0) for item in snapshot.values()), default=None)
|
||||
if snapshot_time is None:
|
||||
snapshot_time = last_snapshot_time or time.time()
|
||||
snapshot_data = {
|
||||
'timestamp': time.time(),
|
||||
'timestamp': snapshot_time,
|
||||
'file_count': file_count,
|
||||
'snapshot': snapshot
|
||||
}
|
||||
@@ -120,6 +120,71 @@ class Monitor(metaclass=Singleton):
|
||||
except Exception as e:
|
||||
logger.error(f"保存快照失败: {e}")
|
||||
|
||||
def reset_snapshot(self, storage: str) -> bool:
|
||||
"""
|
||||
重置快照,强制下次扫描时重新建立基准
|
||||
:param storage: 存储名称
|
||||
:return: 是否成功
|
||||
"""
|
||||
try:
|
||||
cache_file = self._snapshot_cache_dir / f"{storage}_snapshot.json"
|
||||
if cache_file.exists():
|
||||
cache_file.unlink()
|
||||
logger.info(f"快照已重置: {storage}")
|
||||
return True
|
||||
logger.debug(f"快照文件不存在,无需重置: {storage}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"重置快照失败: {storage} - {e}")
|
||||
return False
|
||||
|
||||
def force_full_scan(self, storage: str, mon_path: Path) -> bool:
|
||||
"""
|
||||
强制全量扫描并处理所有文件(包括已存在的文件)
|
||||
:param storage: 存储名称
|
||||
:param mon_path: 监控路径
|
||||
:return: 是否成功
|
||||
"""
|
||||
try:
|
||||
logger.info(f"开始强制全量扫描: {storage}:{mon_path}")
|
||||
|
||||
# 生成快照
|
||||
new_snapshot = StorageChain().snapshot_storage(
|
||||
storage=storage,
|
||||
path=mon_path,
|
||||
last_snapshot_time=0 # 全量扫描,不使用增量
|
||||
)
|
||||
|
||||
if new_snapshot is None:
|
||||
logger.warn(f"获取 {storage}:{mon_path} 快照失败")
|
||||
return False
|
||||
|
||||
file_count = len(new_snapshot)
|
||||
logger.info(f"{storage}:{mon_path} 全量扫描完成,发现 {file_count} 个文件")
|
||||
|
||||
# 处理所有文件
|
||||
processed_count = 0
|
||||
for file_path, file_info in new_snapshot.items():
|
||||
try:
|
||||
logger.info(f"处理文件:{file_path}")
|
||||
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
||||
self.__handle_file(storage=storage, event_path=Path(file_path), file_size=file_size)
|
||||
processed_count += 1
|
||||
except Exception as e:
|
||||
logger.error(f"处理文件 {file_path} 失败: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"{storage}:{mon_path} 全量扫描完成,共处理 {processed_count}/{file_count} 个文件")
|
||||
|
||||
# 保存快照
|
||||
self.save_snapshot(storage, new_snapshot, file_count)
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"强制全量扫描失败: {storage}:{mon_path} - {e}")
|
||||
return False
|
||||
|
||||
def load_snapshot(self, storage: str) -> Optional[Dict]:
|
||||
"""
|
||||
从文件加载快照
|
||||
@@ -131,7 +196,9 @@ class Monitor(metaclass=Singleton):
|
||||
if cache_file.exists():
|
||||
with open(cache_file, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
logger.debug(f"成功加载快照: {cache_file}, 包含 {len(data.get('snapshot', {}))} 个文件")
|
||||
return data
|
||||
logger.debug(f"快照文件不存在: {cache_file}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"加载快照失败: {e}")
|
||||
@@ -327,6 +394,14 @@ class Monitor(metaclass=Singleton):
|
||||
capture_output=True, text=True, timeout=5)
|
||||
if result.returncode == 0:
|
||||
output = result.stdout.lower()
|
||||
# 以下本地文件系统含有fuse关键字
|
||||
local_fs = [
|
||||
"fuse.shfs", # Unraid
|
||||
"zfuse.zfsv", # 极空间(zfuse.zfsv2、zfuse.zfsv3、...)
|
||||
# TBD
|
||||
]
|
||||
if any(fs in output for fs in local_fs):
|
||||
return False
|
||||
network_fs = ['nfs', 'cifs', 'smbfs', 'fuse', 'sshfs', 'ftpfs']
|
||||
return any(fs in output for fs in network_fs)
|
||||
elif system == 'Darwin':
|
||||
@@ -364,6 +439,7 @@ class Monitor(metaclass=Singleton):
|
||||
self._scheduler = BackgroundScheduler(timezone=settings.TZ)
|
||||
|
||||
messagehelper = MessageHelper()
|
||||
mon_storages = {}
|
||||
for mon_dir in monitor_dirs:
|
||||
if not mon_dir.library_path:
|
||||
logger.warn(f"跳过监控配置 {mon_dir.download_path}:未设置媒体库目录")
|
||||
@@ -448,28 +524,33 @@ class Monitor(metaclass=Singleton):
|
||||
|
||||
messagehelper.put(f"启动本地目录监控失败: {mon_path}\n错误: {err_msg}", title="目录监控")
|
||||
else:
|
||||
# 远程目录监控 - 使用智能间隔
|
||||
# 先尝试加载已有快照获取文件数量
|
||||
snapshot_data = self.load_snapshot(mon_dir.storage)
|
||||
file_count = snapshot_data.get('file_count', 0) if snapshot_data else 0
|
||||
interval = self.adjust_monitor_interval(file_count)
|
||||
if not mon_storages.get(mon_dir.storage):
|
||||
mon_storages[mon_dir.storage] = []
|
||||
mon_storages[mon_dir.storage].append(mon_path)
|
||||
|
||||
logger.info(f"正在启动远程目录监控: {mon_path} [{mon_dir.storage}]")
|
||||
logger.info("*** 重要提示:远程目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
|
||||
logger.info(f"预估文件数量: {file_count}, 监控间隔: {interval}分钟")
|
||||
for storage, paths in mon_storages.items():
|
||||
# 远程目录监控 - 使用智能间隔
|
||||
# 先尝试加载已有快照获取文件数量
|
||||
snapshot_data = self.load_snapshot(storage)
|
||||
file_count = snapshot_data.get('file_count', 0) if snapshot_data else 0
|
||||
interval = self.adjust_monitor_interval(file_count)
|
||||
for path in paths:
|
||||
logger.info(f"正在启动远程目录监控: {path} [{storage}]")
|
||||
logger.info("*** 重要提示:远程目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
|
||||
logger.info(f"预估文件数量: {file_count}, 监控间隔: {interval}分钟")
|
||||
|
||||
self._scheduler.add_job(
|
||||
self.polling_observer,
|
||||
'interval',
|
||||
minutes=interval,
|
||||
kwargs={
|
||||
'storage': mon_dir.storage,
|
||||
'mon_path': mon_path
|
||||
},
|
||||
id=f"monitor_{mon_dir.storage}_{mon_dir.download_path}",
|
||||
replace_existing=True
|
||||
)
|
||||
logger.info(f"✓ 远程目录监控已启动: {mon_path} [间隔: {interval}分钟]")
|
||||
self._scheduler.add_job(
|
||||
self.polling_observer,
|
||||
'interval',
|
||||
minutes=interval,
|
||||
kwargs={
|
||||
'storage': storage,
|
||||
'mon_paths': paths
|
||||
},
|
||||
id=f"monitor_{storage}",
|
||||
replace_existing=True
|
||||
)
|
||||
logger.info(f"✓ 远程目录监控已启动: [间隔: {interval}分钟]")
|
||||
|
||||
# 启动定时服务
|
||||
if self._scheduler.get_jobs():
|
||||
@@ -540,34 +621,38 @@ class Monitor(metaclass=Singleton):
|
||||
logger.debug(f"导入 {module_name}.{class_name} 失败: {e}")
|
||||
return None
|
||||
|
||||
def polling_observer(self, storage: str, mon_path: Path):
|
||||
def polling_observer(self, storage: str, mon_paths: List[Path]):
|
||||
"""
|
||||
轮询监控(改进版)
|
||||
"""
|
||||
with snapshot_lock:
|
||||
try:
|
||||
logger.debug(f"开始对 {storage}:{mon_path} 进行快照...")
|
||||
|
||||
# 加载上次快照数据
|
||||
old_snapshot_data = self.load_snapshot(storage)
|
||||
old_snapshot = old_snapshot_data.get('snapshot', {}) if old_snapshot_data else {}
|
||||
last_snapshot_time = old_snapshot_data.get('timestamp', 0) if old_snapshot_data else 0
|
||||
|
||||
# 生成新快照(增量模式)
|
||||
new_snapshot = StorageChain().snapshot_storage(
|
||||
storage=storage,
|
||||
path=mon_path,
|
||||
last_snapshot_time=last_snapshot_time
|
||||
)
|
||||
# 判断是否为首次快照:检查快照文件是否存在且有效
|
||||
is_first_snapshot = old_snapshot_data is None
|
||||
new_snapshot = {}
|
||||
for mon_path in mon_paths:
|
||||
logger.debug(f"开始对 {storage}:{mon_path} 进行快照...")
|
||||
|
||||
if new_snapshot is None:
|
||||
logger.warn(f"获取 {storage}:{mon_path} 快照失败")
|
||||
return
|
||||
# 生成新快照(增量模式)
|
||||
snapshot = StorageChain().snapshot_storage(
|
||||
storage=storage,
|
||||
path=mon_path,
|
||||
last_snapshot_time=last_snapshot_time
|
||||
)
|
||||
|
||||
if snapshot is None:
|
||||
logger.warn(f"获取 {storage}:{mon_path} 快照失败")
|
||||
continue
|
||||
new_snapshot.update(snapshot)
|
||||
file_count = len(snapshot)
|
||||
logger.info(f"{storage}:{mon_path} 快照完成,发现 {file_count} 个文件")
|
||||
file_count = len(new_snapshot)
|
||||
logger.info(f"{storage}:{mon_path} 快照完成,发现 {file_count} 个文件")
|
||||
|
||||
if old_snapshot:
|
||||
if not is_first_snapshot:
|
||||
# 比较快照找出变化
|
||||
changes = self.compare_snapshots(old_snapshot, new_snapshot)
|
||||
|
||||
@@ -587,23 +672,23 @@ class Monitor(metaclass=Singleton):
|
||||
|
||||
if changes['added'] or changes['modified']:
|
||||
logger.info(
|
||||
f"{storage}:{mon_path} 发现 {len(changes['added'])} 个新增文件,{len(changes['modified'])} 个修改文件")
|
||||
f"{storage} 发现 {len(changes['added'])} 个新增文件,{len(changes['modified'])} 个修改文件")
|
||||
else:
|
||||
logger.debug(f"{storage}:{mon_path} 无文件变化")
|
||||
logger.debug(f"{storage} 无文件变化")
|
||||
else:
|
||||
logger.info(f"{storage}:{mon_path} 首次快照完成,共 {file_count} 个文件")
|
||||
logger.info(f"{storage} 首次快照完成,共 {file_count} 个文件")
|
||||
logger.info("*** 首次快照仅建立基准,不会处理现有文件。后续监控将处理新增和修改的文件 ***")
|
||||
|
||||
# 保存新快照
|
||||
self.save_snapshot(storage, new_snapshot, file_count)
|
||||
self.save_snapshot(storage, new_snapshot, file_count, last_snapshot_time)
|
||||
|
||||
# 动态调整监控间隔
|
||||
new_interval = self.adjust_monitor_interval(file_count)
|
||||
current_job = self._scheduler.get_job(f"monitor_{storage}_{mon_path}")
|
||||
current_job = self._scheduler.get_job(f"monitor_{storage}")
|
||||
if current_job and current_job.trigger.interval.total_seconds() / 60 != new_interval:
|
||||
# 重新安排任务
|
||||
self._scheduler.modify_job(
|
||||
f"monitor_{storage}_{mon_path}",
|
||||
f"monitor_{storage}",
|
||||
trigger='interval',
|
||||
minutes=new_interval
|
||||
)
|
||||
@@ -639,7 +724,7 @@ class Monitor(metaclass=Singleton):
|
||||
"""
|
||||
判断是否蓝光原盘目录内的子目录或文件
|
||||
"""
|
||||
return True if re.search(r"BDMV[/\\]STREAM", str(_path), re.IGNORECASE) else False
|
||||
return True if re.search(r"BDMV/STREAM", _path.as_posix(), re.IGNORECASE) else False
|
||||
|
||||
def __get_bluray_dir(_path: Path) -> Optional[Path]:
|
||||
"""
|
||||
@@ -670,7 +755,7 @@ class Monitor(metaclass=Singleton):
|
||||
TransferChain().do_transfer(
|
||||
fileitem=FileItem(
|
||||
storage=storage,
|
||||
path=str(event_path).replace("\\", "/"),
|
||||
path=event_path.as_posix(),
|
||||
type="file",
|
||||
name=event_path.name,
|
||||
basename=event_path.stem,
|
||||
|
||||
@@ -40,20 +40,20 @@ class Scheduler(metaclass=Singleton):
|
||||
"""
|
||||
定时任务管理
|
||||
"""
|
||||
# 定时服务
|
||||
_scheduler = None
|
||||
# 退出事件
|
||||
_event = threading.Event()
|
||||
# 锁
|
||||
_lock = threading.RLock()
|
||||
# 各服务的运行状态
|
||||
_jobs = {}
|
||||
# 用户认证失败次数
|
||||
_auth_count = 0
|
||||
# 用户认证失败消息发送
|
||||
_auth_message = False
|
||||
|
||||
def __init__(self):
|
||||
# 定时服务
|
||||
self._scheduler = None
|
||||
# 退出事件
|
||||
self._event = threading.Event()
|
||||
# 锁
|
||||
self._lock = threading.RLock()
|
||||
# 各服务的运行状态
|
||||
self._jobs = {}
|
||||
# 用户认证失败次数
|
||||
self._auth_count = 0
|
||||
# 用户认证失败消息发送
|
||||
self._auth_message = False
|
||||
self.init()
|
||||
|
||||
@eventmanager.register(EventType.ConfigChanged)
|
||||
@@ -432,7 +432,7 @@ class Scheduler(metaclass=Singleton):
|
||||
"""
|
||||
初始化工作流定时服务
|
||||
"""
|
||||
for workflow in WorkflowChain().get_workflows() or []:
|
||||
for workflow in WorkflowChain().get_timer_workflows() or []:
|
||||
self.update_workflow_job(workflow)
|
||||
|
||||
def remove_workflow_job(self, workflow: Workflow):
|
||||
@@ -443,7 +443,7 @@ class Scheduler(metaclass=Singleton):
|
||||
return
|
||||
with self._lock:
|
||||
job_id = f"workflow-{workflow.id}"
|
||||
service = self._jobs.pop(job_id, None)
|
||||
service = self._jobs.pop(job_id, {})
|
||||
if not service:
|
||||
return
|
||||
try:
|
||||
|
||||
@@ -214,7 +214,7 @@ class ResourceDownloadEventData(ChainEventData):
|
||||
channel: Optional[MessageChannel] = Field(None, description="通知渠道")
|
||||
origin: Optional[str] = Field(None, description="来源")
|
||||
downloader: Optional[str] = Field(None, description="下载器")
|
||||
options: Optional[dict] = Field(None, description="其他参数")
|
||||
options: Optional[dict] = Field(default={}, description="其他参数")
|
||||
|
||||
# 输出参数
|
||||
cancel: bool = Field(default=False, description="是否取消下载")
|
||||
|
||||
@@ -72,6 +72,8 @@ class MediaServerLibrary(BaseModel):
|
||||
image_list: Optional[List[str]] = None
|
||||
# 跳转链接
|
||||
link: Optional[str] = None
|
||||
# 服务器类型
|
||||
server_type: Optional[str] = None
|
||||
|
||||
|
||||
class MediaServerItemUserState(BaseModel):
|
||||
@@ -175,3 +177,4 @@ class MediaServerPlayItem(BaseModel):
|
||||
link: Optional[str] = None
|
||||
percent: Optional[float] = None
|
||||
BackdropImageTags: Optional[list] = Field(default_factory=list)
|
||||
server_type: Optional[str] = None
|
||||
|
||||
@@ -138,6 +138,15 @@ class SubscribeShare(BaseModel):
|
||||
count: Optional[int] = 0
|
||||
|
||||
|
||||
class SubscribeShareStatistics(BaseModel):
|
||||
# 分享人
|
||||
share_user: Optional[str] = None
|
||||
# 分享数量
|
||||
share_count: Optional[int] = 0
|
||||
# 总复用人次
|
||||
total_reuse_count: Optional[int] = 0
|
||||
|
||||
|
||||
class SubscribeDownloadFileInfo(BaseModel):
|
||||
# 种子名称
|
||||
torrent_title: Optional[str] = None
|
||||
|
||||
@@ -65,6 +65,38 @@ class EventType(Enum):
|
||||
ConfigChanged = "config.updated"
|
||||
# 消息交互动作
|
||||
MessageAction = "message.action"
|
||||
# 执行工作流
|
||||
WorkflowExecute = "workflow.execute"
|
||||
|
||||
|
||||
# EventType中文名称翻译字典
|
||||
EVENT_TYPE_NAMES = {
|
||||
EventType.PluginReload: "插件重载",
|
||||
EventType.PluginAction: "触发插件动作",
|
||||
EventType.PluginTriggered: "触发插件事件",
|
||||
EventType.CommandExcute: "执行命令",
|
||||
EventType.SiteDeleted: "站点已删除",
|
||||
EventType.SiteUpdated: "站点已更新",
|
||||
EventType.SiteRefreshed: "站点已刷新",
|
||||
EventType.TransferComplete: "整理完成",
|
||||
EventType.DownloadAdded: "添加下载",
|
||||
EventType.HistoryDeleted: "删除历史记录",
|
||||
EventType.DownloadFileDeleted: "删除下载源文件",
|
||||
EventType.DownloadDeleted: "删除下载任务",
|
||||
EventType.UserMessage: "收到用户消息",
|
||||
EventType.WebhookMessage: "收到Webhook消息",
|
||||
EventType.NoticeMessage: "发送消息通知",
|
||||
EventType.SubscribeAdded: "添加订阅",
|
||||
EventType.SubscribeModified: "订阅已调整",
|
||||
EventType.SubscribeDeleted: "订阅已删除",
|
||||
EventType.SubscribeComplete: "订阅已完成",
|
||||
EventType.SystemError: "系统错误",
|
||||
EventType.MetadataScrape: "刮削元数据",
|
||||
EventType.ModuleReload: "模块重载",
|
||||
EventType.ConfigChanged: "配置项更新",
|
||||
EventType.MessageAction: "消息交互动作",
|
||||
EventType.WorkflowExecute: "执行工作流",
|
||||
}
|
||||
|
||||
|
||||
# 同步链式事件
|
||||
|
||||
@@ -17,6 +17,9 @@ class Workflow(BaseModel):
|
||||
name: Optional[str] = Field(default=None, description="工作流名称")
|
||||
description: Optional[str] = Field(default=None, description="工作流描述")
|
||||
timer: Optional[str] = Field(default=None, description="定时器")
|
||||
trigger_type: Optional[str] = Field(default='timer', description="触发类型:timer-定时触发 event-事件触发 manual-手动触发")
|
||||
event_type: Optional[str] = Field(default=None, description="事件类型(当trigger_type为event时使用)")
|
||||
event_conditions: Optional[dict] = Field(default={}, description="事件条件(JSON格式,用于过滤事件)")
|
||||
state: Optional[str] = Field(default=None, description="状态")
|
||||
current_action: Optional[str] = Field(default=None, description="已执行动作")
|
||||
result: Optional[str] = Field(default=None, description="任务执行结果")
|
||||
@@ -82,3 +85,28 @@ class ActionFlow(BaseModel):
|
||||
source: Optional[str] = Field(default=None, description="源动作")
|
||||
target: Optional[str] = Field(default=None, description="目标动作")
|
||||
animated: Optional[bool] = Field(default=True, description="是否动画流程")
|
||||
|
||||
|
||||
class WorkflowShare(BaseModel):
|
||||
"""
|
||||
工作流分享信息
|
||||
"""
|
||||
id: Optional[int] = Field(default=None, description="分享ID")
|
||||
share_title: Optional[str] = Field(default=None, description="分享标题")
|
||||
share_comment: Optional[str] = Field(default=None, description="分享说明")
|
||||
share_user: Optional[str] = Field(default=None, description="分享人")
|
||||
share_uid: Optional[str] = Field(default=None, description="分享人唯一ID")
|
||||
name: Optional[str] = Field(default=None, description="工作流名称")
|
||||
description: Optional[str] = Field(default=None, description="工作流描述")
|
||||
timer: Optional[str] = Field(default=None, description="定时器")
|
||||
trigger_type: Optional[str] = Field(default=None, description="触发类型")
|
||||
event_type: Optional[str] = Field(default=None, description="事件类型")
|
||||
event_conditions: Optional[str] = Field(default=None, description="事件条件")
|
||||
actions: Optional[str] = Field(default=None, description="任务列表(JSON字符串)")
|
||||
flows: Optional[str] = Field(default=None, description="任务流(JSON字符串)")
|
||||
context: Optional[str] = Field(default=None, description="执行上下文(JSON字符串)")
|
||||
date: Optional[str] = Field(default=None, description="分享时间")
|
||||
count: Optional[int] = Field(default=0, description="复用人次")
|
||||
|
||||
class Config:
|
||||
orm_mode = True
|
||||
|
||||
@@ -5,7 +5,6 @@ from fastapi import FastAPI
|
||||
|
||||
from app.chain.system import SystemChain
|
||||
from app.startup.command_initializer import init_command, stop_command, restart_command
|
||||
from app.startup.memory_initializer import init_memory_manager, stop_memory_manager
|
||||
from app.startup.modules_initializer import init_modules, stop_modules
|
||||
from app.startup.monitor_initializer import stop_monitor, init_monitor
|
||||
from app.startup.plugins_initializer import init_plugins, stop_plugins, sync_plugins
|
||||
@@ -52,8 +51,6 @@ async def lifespan(app: FastAPI):
|
||||
init_command()
|
||||
# 初始化工作流
|
||||
init_workflow()
|
||||
# 初始化内存管理
|
||||
init_memory_manager()
|
||||
# 插件同步到本地
|
||||
sync_plugins_task = asyncio.create_task(init_extra())
|
||||
try:
|
||||
@@ -71,8 +68,6 @@ async def lifespan(app: FastAPI):
|
||||
print(str(e))
|
||||
# 备份插件
|
||||
SystemChain().backup_plugins()
|
||||
# 停止内存管理器
|
||||
stop_memory_manager()
|
||||
# 停止工作流
|
||||
stop_workflow()
|
||||
# 停止命令
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
from app.helper.memory import MemoryHelper
|
||||
|
||||
|
||||
def init_memory_manager():
|
||||
"""
|
||||
初始化内存监控器
|
||||
"""
|
||||
MemoryHelper().start_monitoring()
|
||||
|
||||
|
||||
def stop_memory_manager():
|
||||
"""
|
||||
停止内存监控器
|
||||
"""
|
||||
MemoryHelper().stop_monitoring()
|
||||
@@ -1,15 +1,16 @@
|
||||
from app.core.workflow import WorkFlowManager
|
||||
from app.chain.workflow import WorkflowChain
|
||||
|
||||
|
||||
def init_workflow():
|
||||
"""
|
||||
初始化动作
|
||||
初始化工作流
|
||||
"""
|
||||
WorkFlowManager()
|
||||
|
||||
|
||||
def stop_workflow():
|
||||
"""
|
||||
停止动作
|
||||
停止工作流
|
||||
"""
|
||||
WorkFlowManager().stop()
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import sys
|
||||
import re
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
import chardet
|
||||
@@ -8,6 +10,7 @@ import urllib3
|
||||
from requests import Response, Session
|
||||
from urllib3.exceptions import InsecureRequestWarning
|
||||
|
||||
from app.core.config import settings
|
||||
from app.log import logger
|
||||
|
||||
urllib3.disable_warnings(InsecureRequestWarning)
|
||||
@@ -86,6 +89,7 @@ class AutoCloseResponse:
|
||||
def __exit__(self, *args):
|
||||
self.close()
|
||||
|
||||
|
||||
class RequestUtils:
|
||||
|
||||
def __init__(self,
|
||||
@@ -106,6 +110,10 @@ class RequestUtils:
|
||||
if headers:
|
||||
self._headers = headers
|
||||
else:
|
||||
if ua and ua == settings.USER_AGENT:
|
||||
caller_name = self.__get_caller()
|
||||
if caller_name:
|
||||
ua = f"{settings.USER_AGENT} Plugin/{caller_name}"
|
||||
self._headers = {
|
||||
"User-Agent": ua,
|
||||
"Content-Type": content_type,
|
||||
@@ -120,6 +128,43 @@ class RequestUtils:
|
||||
else:
|
||||
self._cookies = None
|
||||
|
||||
@staticmethod
|
||||
def __get_caller():
|
||||
"""
|
||||
获取调用者的名称,识别是否为插件调用
|
||||
"""
|
||||
# 调用者名称
|
||||
caller_name = None
|
||||
|
||||
try:
|
||||
frame = sys._getframe(3) # noqa
|
||||
except (AttributeError, ValueError):
|
||||
return None
|
||||
|
||||
while frame:
|
||||
filepath = Path(frame.f_code.co_filename)
|
||||
parts = filepath.parts
|
||||
if "app" in parts:
|
||||
if not caller_name and "plugins" in parts:
|
||||
try:
|
||||
plugins_index = parts.index("plugins")
|
||||
if plugins_index + 1 < len(parts):
|
||||
plugin_candidate = parts[plugins_index + 1]
|
||||
if plugin_candidate != "__init__.py":
|
||||
caller_name = plugin_candidate
|
||||
break
|
||||
except ValueError:
|
||||
pass
|
||||
if "main.py" in parts:
|
||||
break
|
||||
elif len(parts) != 1:
|
||||
break
|
||||
try:
|
||||
frame = frame.f_back
|
||||
except AttributeError:
|
||||
break
|
||||
return caller_name
|
||||
|
||||
def request(self, method: str, url: str, raise_exception: bool = False, **kwargs) -> Optional[Response]:
|
||||
"""
|
||||
发起HTTP请求
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import abc
|
||||
import threading
|
||||
import weakref
|
||||
|
||||
|
||||
class Singleton(abc.ABCMeta, type):
|
||||
@@ -40,3 +42,17 @@ class AbstractSingletonClass(abc.ABC, metaclass=SingletonClass):
|
||||
抽像类单例模式(按类)
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class WeakSingleton(abc.ABCMeta, type):
|
||||
"""
|
||||
弱引用单例模式 - 当没有强引用时自动清理
|
||||
"""
|
||||
_instances: weakref.WeakKeyDictionary = weakref.WeakKeyDictionary()
|
||||
_lock = threading.RLock()
|
||||
|
||||
def __call__(cls, *args, **kwargs):
|
||||
with cls._lock:
|
||||
if cls not in cls._instances:
|
||||
cls._instances[cls] = super().__call__(*args, **kwargs)
|
||||
return cls._instances[cls]
|
||||
|
||||
@@ -13,27 +13,31 @@ class SiteUtils:
|
||||
:return:
|
||||
"""
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return False
|
||||
# 存在明显的密码输入框,说明未登录
|
||||
if html.xpath("//input[@type='password']"):
|
||||
return False
|
||||
# 是否存在登出和用户面板等链接
|
||||
xpaths = [
|
||||
'//a[contains(@href, "logout")'
|
||||
' or contains(@data-url, "logout")'
|
||||
' or contains(@href, "mybonus") '
|
||||
' or contains(@onclick, "logout")'
|
||||
' or contains(@href, "usercp")'
|
||||
' or contains(@lay-on, "logout")]',
|
||||
'//form[contains(@action, "logout")]',
|
||||
'//div[@class="user-info-side"]',
|
||||
'//a[@id="myitem"]'
|
||||
]
|
||||
for xpath in xpaths:
|
||||
if html.xpath(xpath):
|
||||
return True
|
||||
return False
|
||||
# 存在明显的密码输入框,说明未登录
|
||||
if html.xpath("//input[@type='password']"):
|
||||
return False
|
||||
# 是否存在登出和用户面板等链接
|
||||
xpaths = [
|
||||
'//a[contains(@href, "logout")'
|
||||
' or contains(@data-url, "logout")'
|
||||
' or contains(@href, "mybonus") '
|
||||
' or contains(@onclick, "logout")'
|
||||
' or contains(@href, "usercp")'
|
||||
' or contains(@lay-on, "logout")]',
|
||||
'//form[contains(@action, "logout")]',
|
||||
'//div[@class="user-info-side"]',
|
||||
'//a[@id="myitem"]'
|
||||
]
|
||||
for xpath in xpaths:
|
||||
if html.xpath(xpath):
|
||||
return True
|
||||
return False
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
@classmethod
|
||||
def is_checkin(cls, html_text: str) -> bool:
|
||||
@@ -42,24 +46,27 @@ class SiteUtils:
|
||||
:return True已签到 False未签到
|
||||
"""
|
||||
html = etree.HTML(html_text)
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return False
|
||||
# 站点签到支持的识别XPATH
|
||||
xpaths = [
|
||||
'//a[@id="signed"]',
|
||||
'//a[contains(@href, "attendance")]',
|
||||
'//a[contains(text(), "签到")]',
|
||||
'//a/b[contains(text(), "签 到")]',
|
||||
'//span[@id="sign_in"]/a',
|
||||
'//a[contains(@href, "addbonus")]',
|
||||
'//input[@class="dt_button"][contains(@value, "打卡")]',
|
||||
'//a[contains(@href, "sign_in")]',
|
||||
'//a[contains(@onclick, "do_signin")]',
|
||||
'//a[@id="do-attendance"]',
|
||||
'//shark-icon-button[@href="attendance.php"]'
|
||||
]
|
||||
for xpath in xpaths:
|
||||
if html.xpath(xpath):
|
||||
try:
|
||||
if not StringUtils.is_valid_html_element(html):
|
||||
return False
|
||||
|
||||
return True
|
||||
# 站点签到支持的识别XPATH
|
||||
xpaths = [
|
||||
'//a[@id="signed"]',
|
||||
'//a[contains(@href, "attendance")]',
|
||||
'//a[contains(text(), "签到")]',
|
||||
'//a/b[contains(text(), "签 到")]',
|
||||
'//span[@id="sign_in"]/a',
|
||||
'//a[contains(@href, "addbonus")]',
|
||||
'//input[@class="dt_button"][contains(@value, "打卡")]',
|
||||
'//a[contains(@href, "sign_in")]',
|
||||
'//a[contains(@onclick, "do_signin")]',
|
||||
'//a[@id="do-attendance"]',
|
||||
'//shark-icon-button[@href="attendance.php"]'
|
||||
]
|
||||
for xpath in xpaths:
|
||||
if html.xpath(xpath):
|
||||
return False
|
||||
return True
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
@@ -68,35 +68,57 @@ class SystemUtils:
|
||||
"""
|
||||
if SystemUtils.is_windows():
|
||||
return False
|
||||
return True if "synology" in SystemUtils.execute('uname -a') else False
|
||||
return "synology" in SystemUtils.execute('uname -a')
|
||||
|
||||
@staticmethod
|
||||
def is_windows() -> bool:
|
||||
"""
|
||||
判断是否为Windows系统
|
||||
"""
|
||||
return True if os.name == "nt" else False
|
||||
return os.name == "nt"
|
||||
|
||||
@staticmethod
|
||||
def is_frozen() -> bool:
|
||||
"""
|
||||
判断是否为冻结的二进制文件
|
||||
"""
|
||||
return True if getattr(sys, 'frozen', False) else False
|
||||
return getattr(sys, 'frozen', False)
|
||||
|
||||
@staticmethod
|
||||
def is_macos() -> bool:
|
||||
"""
|
||||
判断是否为MacOS系统
|
||||
"""
|
||||
return True if platform.system() == 'Darwin' else False
|
||||
return platform.system() == 'Darwin'
|
||||
|
||||
@staticmethod
|
||||
def is_aarch64() -> bool:
|
||||
"""
|
||||
判断是否为ARM64架构
|
||||
"""
|
||||
return True if platform.machine() == 'aarch64' else False
|
||||
return platform.machine().lower() in ('aarch64', 'arm64')
|
||||
|
||||
@staticmethod
|
||||
def is_aarch() -> bool:
|
||||
"""
|
||||
判断是否为ARM32架构
|
||||
"""
|
||||
arch_name = platform.machine().lower()
|
||||
return arch_name.startswith(('arm', 'aarch')) and arch_name not in ('aarch64', 'arm64')
|
||||
|
||||
@staticmethod
|
||||
def is_x86_64() -> bool:
|
||||
"""
|
||||
判断是否为AMD64架构
|
||||
"""
|
||||
return platform.machine().lower() in ('amd64', 'x86_64')
|
||||
|
||||
@staticmethod
|
||||
def is_x86_32() -> bool:
|
||||
"""
|
||||
判断是否为AMD32架构
|
||||
"""
|
||||
return platform.machine().lower() in ('i386', 'i686', 'x86', '386', 'x86_32')
|
||||
|
||||
@staticmethod
|
||||
def platform() -> str:
|
||||
@@ -112,6 +134,22 @@ class SystemUtils:
|
||||
else:
|
||||
return "Linux"
|
||||
|
||||
@staticmethod
|
||||
def cpu_arch() -> str:
|
||||
"""
|
||||
获取CPU架构
|
||||
"""
|
||||
if SystemUtils.is_x86_64():
|
||||
return "x86_64"
|
||||
elif SystemUtils.is_x86_32():
|
||||
return "x86_32"
|
||||
elif SystemUtils.is_aarch64():
|
||||
return "Arm64"
|
||||
elif SystemUtils.is_aarch():
|
||||
return "Arm32"
|
||||
else:
|
||||
return platform.machine()
|
||||
|
||||
@staticmethod
|
||||
def copy(src: Path, dest: Path) -> Tuple[int, str]:
|
||||
"""
|
||||
|
||||
37
database/versions/4666ce24a443_2_1_8.py
Normal file
37
database/versions/4666ce24a443_2_1_8.py
Normal file
@@ -0,0 +1,37 @@
|
||||
"""2.1.8
|
||||
|
||||
Revision ID: 4666ce24a443
|
||||
Revises: 3891a5e722a1
|
||||
Create Date: 2025-07-22 13:54:04.196126
|
||||
|
||||
"""
|
||||
import contextlib
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '4666ce24a443'
|
||||
down_revision = '3891a5e722a1'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
with contextlib.suppress(Exception):
|
||||
# 添加触发类型字段
|
||||
op.add_column('workflow', sa.Column('trigger_type', sa.String(), nullable=True, default='timer'))
|
||||
|
||||
with contextlib.suppress(Exception):
|
||||
# 添加事件类型字段
|
||||
op.add_column('workflow', sa.Column('event_type', sa.String(), nullable=True))
|
||||
|
||||
with contextlib.suppress(Exception):
|
||||
# 添加事件条件字段
|
||||
op.add_column('workflow', sa.Column('event_conditions', sa.JSON(), nullable=True, default={}))
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
@@ -28,6 +28,7 @@ RUN apt-get update -y \
|
||||
rsync \
|
||||
ffmpeg \
|
||||
nano \
|
||||
&& dpkg-reconfigure --frontend noninteractive tzdata \
|
||||
&& \
|
||||
if [ "$(uname -m)" = "x86_64" ]; \
|
||||
then ln -s /usr/lib/x86_64-linux-musl/libc.so /lib/libc.musl-x86_64.so.1; \
|
||||
@@ -85,4 +86,4 @@ RUN cp -f /app/docker/nginx.common.conf /etc/nginx/common.conf \
|
||||
&& rm -rf /tmp/*
|
||||
EXPOSE 3000
|
||||
VOLUME [ "${CONFIG_DIR}" ]
|
||||
ENTRYPOINT [ "/entrypoint.sh" ]
|
||||
ENTRYPOINT [ "/entrypoint.sh" ]
|
||||
|
||||
@@ -70,3 +70,4 @@ tqdm~=4.67.1
|
||||
setuptools~=78.1.0
|
||||
pympler~=1.1
|
||||
smbprotocol~=1.15.0
|
||||
setproctitle~=1.3.6
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
APP_VERSION = 'v2.6.1'
|
||||
FRONTEND_VERSION = 'v2.6.1'
|
||||
APP_VERSION = 'v2.6.8'
|
||||
FRONTEND_VERSION = 'v2.6.8'
|
||||
|
||||
Reference in New Issue
Block a user