merge db oper

This commit is contained in:
jxxghp
2024-09-26 14:13:29 +08:00
parent c6abb1f9f1
commit 46fb52fff9
8 changed files with 109 additions and 154 deletions
+7 -11
View File
@@ -13,9 +13,7 @@ from app.core.config import settings
from app.core.event import eventmanager, Event, EventManager from app.core.event import eventmanager, Event, EventManager
from app.db.models.site import Site from app.db.models.site import Site
from app.db.site_oper import SiteOper from app.db.site_oper import SiteOper
from app.db.siteicon_oper import SiteIconOper
from app.db.systemconfig_oper import SystemConfigOper from app.db.systemconfig_oper import SystemConfigOper
from app.db.sitestatistic_oper import SiteStatisticOper
from app.helper.browser import PlaywrightHelper from app.helper.browser import PlaywrightHelper
from app.helper.cloudflare import under_challenge from app.helper.cloudflare import under_challenge
from app.helper.cookie import CookieHelper from app.helper.cookie import CookieHelper
@@ -39,14 +37,12 @@ class SiteChain(ChainBase):
def __init__(self): def __init__(self):
super().__init__() super().__init__()
self.siteoper = SiteOper() self.siteoper = SiteOper()
self.siteiconoper = SiteIconOper()
self.siteshelper = SitesHelper() self.siteshelper = SitesHelper()
self.rsshelper = RssHelper() self.rsshelper = RssHelper()
self.cookiehelper = CookieHelper() self.cookiehelper = CookieHelper()
self.message = MessageHelper() self.message = MessageHelper()
self.cookiecloud = CookieCloudHelper() self.cookiecloud = CookieCloudHelper()
self.systemconfig = SystemConfigOper() self.systemconfig = SystemConfigOper()
self.sitestatistic = SiteStatisticOper()
# 特殊站点登录验证 # 特殊站点登录验证
self.special_site_test = { self.special_site_test = {
@@ -362,17 +358,17 @@ class SiteChain(ChainBase):
logger.warn(f"站点 {domain} 索引器不存在!") logger.warn(f"站点 {domain} 索引器不存在!")
return return
# 查询站点图标 # 查询站点图标
site_icon = self.siteiconoper.get_by_domain(domain) site_icon = self.siteoper.get_icon_by_domain(domain)
if not site_icon or not site_icon.base64: if not site_icon or not site_icon.base64:
logger.info(f"开始缓存站点 {indexer.get('name')} 图标 ...") logger.info(f"开始缓存站点 {indexer.get('name')} 图标 ...")
icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"), icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"),
cookie=cookie, cookie=cookie,
ua=settings.USER_AGENT) ua=settings.USER_AGENT)
if icon_url: if icon_url:
self.siteiconoper.update_icon(name=indexer.get("name"), self.siteoper.update_icon(name=indexer.get("name"),
domain=domain, domain=domain,
icon_url=icon_url, icon_url=icon_url,
icon_base64=icon_base64) icon_base64=icon_base64)
logger.info(f"缓存站点 {indexer.get('name')} 图标成功") logger.info(f"缓存站点 {indexer.get('name')} 图标成功")
else: else:
logger.warn(f"缓存站点 {indexer.get('name')} 图标失败") logger.warn(f"缓存站点 {indexer.get('name')} 图标失败")
@@ -443,9 +439,9 @@ class SiteChain(ChainBase):
# 统计 # 统计
seconds = (datetime.now() - start_time).seconds seconds = (datetime.now() - start_time).seconds
if state: if state:
self.sitestatistic.success(domain=domain, seconds=seconds) self.siteoper.success(domain=domain, seconds=seconds)
else: else:
self.sitestatistic.fail(domain) self.siteoper.fail(domain)
return state, message return state, message
except Exception as e: except Exception as e:
return False, f"{str(e)}" return False, f"{str(e)}"
+1 -3
View File
@@ -20,7 +20,6 @@ from app.db.downloadhistory_oper import DownloadHistoryOper
from app.db.models.subscribe import Subscribe from app.db.models.subscribe import Subscribe
from app.db.site_oper import SiteOper from app.db.site_oper import SiteOper
from app.db.subscribe_oper import SubscribeOper from app.db.subscribe_oper import SubscribeOper
from app.db.subscribehistory_oper import SubscribeHistoryOper
from app.db.systemconfig_oper import SystemConfigOper from app.db.systemconfig_oper import SystemConfigOper
from app.helper.message import MessageHelper from app.helper.message import MessageHelper
from app.helper.subscribe import SubscribeHelper from app.helper.subscribe import SubscribeHelper
@@ -41,7 +40,6 @@ class SubscribeChain(ChainBase):
self.downloadhis = DownloadHistoryOper() self.downloadhis = DownloadHistoryOper()
self.searchchain = SearchChain() self.searchchain = SearchChain()
self.subscribeoper = SubscribeOper() self.subscribeoper = SubscribeOper()
self.subscribehistoryoper = SubscribeHistoryOper()
self.subscribehelper = SubscribeHelper() self.subscribehelper = SubscribeHelper()
self.torrentschain = TorrentsChain() self.torrentschain = TorrentsChain()
self.mediachain = MediaChain() self.mediachain = MediaChain()
@@ -898,7 +896,7 @@ class SubscribeChain(ChainBase):
msgstr = "洗版" msgstr = "洗版"
logger.info(f'{mediainfo.title_year} 完成{msgstr}') logger.info(f'{mediainfo.title_year} 完成{msgstr}')
# 新增订阅历史 # 新增订阅历史
self.subscribehistoryoper.add(**subscribe.to_dict()) self.subscribeoper.add_history(**subscribe.to_dict())
# 删除订阅 # 删除订阅
self.subscribeoper.delete(subscribe.id) self.subscribeoper.delete(subscribe.id)
# 发送通知 # 发送通知
+82
View File
@@ -3,7 +3,9 @@ from datetime import datetime
from typing import Tuple, List from typing import Tuple, List
from app.db import DbOper from app.db import DbOper
from app.db.models import SiteIcon
from app.db.models.site import Site from app.db.models.site import Site
from app.db.models.sitestatistic import SiteStatistic
from app.db.models.siteuserdata import SiteUserData from app.db.models.siteuserdata import SiteUserData
from app.utils.object import ObjectUtils from app.utils.object import ObjectUtils
@@ -139,3 +141,83 @@ class SiteOper(DbOper):
获取站点用户数据 获取站点用户数据
""" """
return SiteUserData.get_by_date(self._db, date) return SiteUserData.get_by_date(self._db, date)
def get_icon_by_domain(self, domain: str) -> SiteIcon:
"""
按域名获取站点图标
"""
return SiteIcon.get_by_domain(self._db, domain)
def update_icon(self, name: str, domain: str, icon_url: str, icon_base64: str) -> bool:
"""
更新站点图标
"""
icon_base64 = f"data:image/ico;base64,{icon_base64}" if icon_base64 else ""
siteicon = self.get_by_domain(domain)
if not siteicon:
SiteIcon(name=name, domain=domain, url=icon_url, base64=icon_base64).create(self._db)
elif icon_base64:
siteicon.update(self._db, {
"url": icon_url,
"base64": icon_base64
})
return True
def success(self, domain: str, seconds: int = None):
"""
站点访问成功
"""
lst_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
sta = SiteStatistic.get_by_domain(self._db, domain)
if sta:
avg_seconds, note = None, {}
if seconds is not None:
note: dict = json.loads(sta.note or "{}")
note[lst_date] = seconds or 1
avg_times = len(note.keys())
if avg_times > 10:
note = dict(sorted(note.items(), key=lambda x: x[0], reverse=True)[:10])
avg_seconds = sum([v for v in note.values()]) // avg_times
sta.update(self._db, {
"success": sta.success + 1,
"seconds": avg_seconds or sta.seconds,
"lst_state": 0,
"lst_mod_date": lst_date,
"note": note or sta.note
})
else:
note = {}
if seconds is not None:
note = {
lst_date: seconds or 1
}
SiteStatistic(
domain=domain,
success=1,
fail=0,
seconds=seconds or 1,
lst_state=0,
lst_mod_date=lst_date,
note=json.dumps(note)
).create(self._db)
def fail(self, domain: str):
"""
站点访问失败
"""
lst_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
sta = SiteStatistic.get_by_domain(self._db, domain)
if sta:
sta.update(self._db, {
"fail": sta.fail + 1,
"lst_state": 1,
"lst_mod_date": lst_date
})
else:
SiteStatistic(
domain=domain,
success=0,
fail=1,
lst_state=1,
lst_mod_date=lst_date
).create(self._db)
-37
View File
@@ -1,37 +0,0 @@
from typing import List
from app.db import DbOper
from app.db.models.siteicon import SiteIcon
class SiteIconOper(DbOper):
"""
站点管理
"""
def list(self) -> List[SiteIcon]:
"""
获取站点图标列表
"""
return SiteIcon.list(self._db)
def get_by_domain(self, domain: str) -> SiteIcon:
"""
按域名获取站点图标
"""
return SiteIcon.get_by_domain(self._db, domain)
def update_icon(self, name: str, domain: str, icon_url: str, icon_base64: str) -> bool:
"""
更新站点图标
"""
icon_base64 = f"data:image/ico;base64,{icon_base64}" if icon_base64 else ""
siteicon = self.get_by_domain(domain)
if not siteicon:
SiteIcon(name=name, domain=domain, url=icon_url, base64=icon_base64).create(self._db)
elif icon_base64:
siteicon.update(self._db, {
"url": icon_url,
"base64": icon_base64
})
return True
-70
View File
@@ -1,70 +0,0 @@
import json
from datetime import datetime
from app.db import DbOper
from app.db.models.sitestatistic import SiteStatistic
class SiteStatisticOper(DbOper):
"""
站点统计管理
"""
def success(self, domain: str, seconds: int = None):
"""
站点访问成功
"""
lst_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
sta = SiteStatistic.get_by_domain(self._db, domain)
if sta:
avg_seconds, note = None, {}
if seconds is not None:
note: dict = json.loads(sta.note or "{}")
note[lst_date] = seconds or 1
avg_times = len(note.keys())
if avg_times > 10:
note = dict(sorted(note.items(), key=lambda x: x[0], reverse=True)[:10])
avg_seconds = sum([v for v in note.values()]) // avg_times
sta.update(self._db, {
"success": sta.success + 1,
"seconds": avg_seconds or sta.seconds,
"lst_state": 0,
"lst_mod_date": lst_date,
"note": note or sta.note
})
else:
note = {}
if seconds is not None:
note = {
lst_date: seconds or 1
}
SiteStatistic(
domain=domain,
success=1,
fail=0,
seconds=seconds or 1,
lst_state=0,
lst_mod_date=lst_date,
note=json.dumps(note)
).create(self._db)
def fail(self, domain: str):
"""
站点访问失败
"""
lst_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
sta = SiteStatistic.get_by_domain(self._db, domain)
if sta:
sta.update(self._db, {
"fail": sta.fail + 1,
"lst_state": 1,
"lst_mod_date": lst_date
})
else:
SiteStatistic(
domain=domain,
success=0,
fail=1,
lst_state=1,
lst_mod_date=lst_date
).create(self._db)
+16
View File
@@ -4,6 +4,7 @@ from typing import Tuple, List
from app.core.context import MediaInfo from app.core.context import MediaInfo
from app.db import DbOper from app.db import DbOper
from app.db.models.subscribe import Subscribe from app.db.models.subscribe import Subscribe
from app.db.models.subscribehistory import SubscribeHistory
class SubscribeOper(DbOper): class SubscribeOper(DbOper):
@@ -102,3 +103,18 @@ class SubscribeOper(DbOper):
获取指定类型的订阅 获取指定类型的订阅
""" """
return Subscribe.list_by_type(self._db, mtype=mtype, days=days) return Subscribe.list_by_type(self._db, mtype=mtype, days=days)
def add_history(self, **kwargs):
"""
新增订阅
"""
# 去除kwargs中 SubscribeHistory 没有的字段
kwargs = {k: v for k, v in kwargs.items() if hasattr(SubscribeHistory, k)}
# 更新完成订阅时间
kwargs.update({"date": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())})
# 去掉主键
if "id" in kwargs:
kwargs.pop("id")
subscribe = SubscribeHistory(**kwargs)
subscribe.create(self._db)
-30
View File
@@ -1,30 +0,0 @@
import time
from app.db import DbOper
from app.db.models.subscribehistory import SubscribeHistory
class SubscribeHistoryOper(DbOper):
"""
订阅历史管理
"""
def add(self, **kwargs):
"""
新增订阅
"""
# 去除kwargs中 SubscribeHistory 没有的字段
kwargs = {k: v for k, v in kwargs.items() if hasattr(SubscribeHistory, k)}
# 更新完成订阅时间
kwargs.update({"date": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())})
# 去掉主键
if "id" in kwargs:
kwargs.pop("id")
subscribe = SubscribeHistory(**kwargs)
subscribe.create(self._db)
def list_by_type(self, mtype: str, page: int = 1, count: int = 30) -> SubscribeHistory:
"""
获取指定类型的订阅
"""
return SubscribeHistory.list_by_type(self._db, mtype=mtype, page=page, count=count)
+3 -3
View File
@@ -5,7 +5,7 @@ from ruamel.yaml import CommentedMap
from app.core.config import settings from app.core.config import settings
from app.core.context import TorrentInfo from app.core.context import TorrentInfo
from app.db.sitestatistic_oper import SiteStatisticOper from app.db.site_oper import SiteOper
from app.helper.module import ModuleHelper from app.helper.module import ModuleHelper
from app.helper.sites import SitesHelper from app.helper.sites import SitesHelper
from app.log import logger from app.log import logger
@@ -160,9 +160,9 @@ class IndexerModule(_ModuleBase):
# 统计索引情况 # 统计索引情况
domain = StringUtils.get_url_domain(site.get("domain")) domain = StringUtils.get_url_domain(site.get("domain"))
if error_flag: if error_flag:
SiteStatisticOper().fail(domain) SiteOper().fail(domain)
else: else:
SiteStatisticOper().success(domain=domain, seconds=seconds) SiteOper().success(domain=domain, seconds=seconds)
# 返回结果 # 返回结果
if not result_array or len(result_array) == 0: if not result_array or len(result_array) == 0: