diff --git a/app/chain/site.py b/app/chain/site.py index a9b38254..f4be1f12 100644 --- a/app/chain/site.py +++ b/app/chain/site.py @@ -65,10 +65,10 @@ class SiteChain(ChainBase): :param site: 站点 :return: 用户数据 """ - userdata = self.run_module("refresh_userdata", site=site) + userdata: SiteUserData = self.run_module("refresh_userdata", site=site) if userdata: self.siteoper.update_userdata(domain=StringUtils.get_url_domain(site.get("domain")), - payload=userdata) + payload=userdata.dict()) return userdata def refresh_userdatas(self) -> None: diff --git a/app/db/site_oper.py b/app/db/site_oper.py index 9120cba4..beb1984f 100644 --- a/app/db/site_oper.py +++ b/app/db/site_oper.py @@ -1,9 +1,11 @@ +import json from datetime import datetime from typing import Tuple, List from app.db import DbOper from app.db.models.site import Site from app.db.models.siteuserdata import SiteUserData +from app.utils.object import ObjectUtils class SiteOper(DbOper): @@ -120,6 +122,9 @@ class SiteOper(DbOper): SiteUserData.update(self._db, payload) else: # 不存在则插入 + for key, value in payload.items(): + if ObjectUtils.is_obj(value): + payload[key] = json.dumps(value) SiteUserData(**payload).create(self._db) return True, "更新站点用户数据成功" diff --git a/app/modules/indexer/__init__.py b/app/modules/indexer/__init__.py index e4ccef76..47c693a1 100644 --- a/app/modules/indexer/__init__.py +++ b/app/modules/indexer/__init__.py @@ -10,6 +10,7 @@ from app.helper.module import ModuleHelper from app.helper.sites import SitesHelper from app.log import logger from app.modules import _ModuleBase +from app.modules.indexer.parser import SiteParserBase from app.modules.indexer.spider import TorrentSpider from app.modules.indexer.spider.haidan import HaiDanSpider from app.modules.indexer.spider.mtorrent import MTorrentSpider @@ -30,8 +31,10 @@ class IndexerModule(_ModuleBase): def init_module(self) -> None: # 加载模块 - self._site_schemas = ModuleHelper.load('app.modules.indexer.parser', - filter_func=lambda _, obj: hasattr(obj, 'schema')) + self._site_schemas = ModuleHelper.load( + 'app.modules.indexer.parser', + filter_func=lambda _, obj: hasattr(obj, 'schema') and getattr(obj, 'schema') is not None) + pass @staticmethod def get_name() -> str: @@ -214,12 +217,12 @@ class IndexerModule(_ModuleBase): :return: 用户数据 """ - def __get_site_obj(): + def __get_site_obj() -> Optional[SiteParserBase]: """ 获取站点解析器 """ for site_schema in self._site_schemas: - if site_schema.schema == site.get("schema"): + if site_schema.schema.value == site.get("schema"): return site_schema( site_name=site.get("name"), url=site.get("url"), @@ -232,13 +235,14 @@ class IndexerModule(_ModuleBase): site_obj = __get_site_obj() if not site_obj: - logger.warn(f"站点 {site.get('name')} 未找到站点解析器: {site.get('schema')}") + if not site.get("public"): + logger.warn(f"站点 {site.get('name')} 未找到站点解析器,schema:{site.get('schema')}") return None # 获取用户数据 - logger.debug(f"站点 {site.get('name')} 开始以 {site.get('schema')} 模型解析") + logger.info(f"站点 {site.get('name')} 开始以 {site.get('schema')} 模型解析数据...") site_obj.parse() - logger.debug(f"站点 {site.get('name')} 解析完成") + logger.debug(f"站点 {site.get('name')} 数据解析完成") return SiteUserData( domain=StringUtils.get_url_domain(site.get("url")), userid=site_obj.userid, @@ -251,11 +255,11 @@ class IndexerModule(_ModuleBase): bonus=site_obj.bonus, seeding=site_obj.seeding, seeding_size=site_obj.seeding_size, - seeding_info=site_obj.seeding_info, + seeding_info=site_obj.seeding_info or [], leeching=site_obj.leeching, leeching_size=site_obj.leeching_size, message_unread=site_obj.message_unread, - message_unread_contents=site_obj.message_unread_contents, + message_unread_contents=site_obj.message_unread_contents or [], updated_at=datetime.now().strftime('%Y-%m-%d'), err_msg=site_obj.err_msg ) diff --git a/app/modules/indexer/parser/__init__.py b/app/modules/indexer/parser/__init__.py index 97ae48d1..e1e82d6f 100644 --- a/app/modules/indexer/parser/__init__.py +++ b/app/modules/indexer/parser/__init__.py @@ -36,7 +36,7 @@ class SiteSchema(Enum): class SiteParserBase(metaclass=ABCMeta): # 站点模版 - schema = SiteSchema.NexusPhp + schema = None # 请求模式 cookie/apikey request_mode = "cookie" @@ -45,7 +45,6 @@ class SiteParserBase(metaclass=ABCMeta): site_cookie: str, apikey: str, token: str, - index_html: str, session: Session = None, ua: str = None, emulate: bool = False, @@ -61,12 +60,11 @@ class SiteParserBase(metaclass=ABCMeta): self._site_domain = __split_url.netloc self._base_url = f"{__split_url.scheme}://{__split_url.netloc}" self._site_cookie = site_cookie - self._index_html = index_html self._session = session if session else None self._ua = ua self._emulate = emulate self._proxy = proxy - + self._index_html = "" # 用户信息 self.username = None self.userid = None @@ -156,6 +154,8 @@ class SiteParserBase(metaclass=ABCMeta): 解析站点信息 :return: """ + # 获取站点首页html + self._index_html = self._get_page_content(url=self._site_url) # 检查是否已经登录 if not self._parse_logged_in(self._index_html): return @@ -194,7 +194,6 @@ class SiteParserBase(metaclass=ABCMeta): ) # 解析用户做种信息 self._parse_seeding_pages() - self.seeding_info = json.dumps(self.seeding_info) def _pase_unread_msgs(self): """