GithubBackup/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-09-05 07:27:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

remove gc

Browse Source
This commit is contained in:
jxxghp
2025-06-08 21:19:15 +08:00
parent 7f8e50f83d
commit 0a20234268
2 changed files with 30 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/helper/browser.py
+2 -6
View File
@@ -1,8 +1,8 @@
from typing import Callable, Any, Optional from typing import Callable, Any, Optional
import gc
from playwright.sync_api import sync_playwright, Page
from cf_clearance import sync_cf_retry, sync_stealth from cf_clearance import sync_cf_retry, sync_stealth
from playwright.sync_api import sync_playwright, Page
from app.log import logger from app.log import logger
@@ -67,8 +67,6 @@ class PlaywrightHelper:
context.close() context.close()
if browser: if browser:
browser.close() browser.close()
# 强制垃圾回收
gc.collect()
except Exception as e: except Exception as e:
logger.error(f"Playwright初始化失败: {str(e)}") logger.error(f"Playwright初始化失败: {str(e)}")
@@ -120,8 +118,6 @@ class PlaywrightHelper:
context.close() context.close()
if browser: if browser:
browser.close() browser.close()
# 强制垃圾回收
gc.collect()
except Exception as e: except Exception as e:
logger.error(f"Playwright初始化失败: {str(e)}") logger.error(f"Playwright初始化失败: {str(e)}")
app/helper/rss.py
+28 -32
View File
@@ -1,4 +1,3 @@
import gc
import re import re
import traceback import traceback
from typing import List, Tuple, Union, Optional from typing import List, Tuple, Union, Optional
@@ -18,11 +17,11 @@ class RssHelper:
""" """
RSS帮助类，解析RSS报文、获取RSS地址等 RSS帮助类，解析RSS报文、获取RSS地址等
""" """
# RSS解析限制配置 # RSS解析限制配置
MAX_RSS_SIZE = 50 * 1024 * 1024 # 50MB最大RSS文件大小 MAX_RSS_SIZE = 50 * 1024 * 1024 # 50MB最大RSS文件大小
MAX_RSS_ITEMS = 1000 # 最大解析条目数 MAX_RSS_ITEMS = 1000 # 最大解析条目数
# 各站点RSS链接获取配置 # 各站点RSS链接获取配置
rss_link_conf = { rss_link_conf = {
"default": { "default": {
@@ -228,7 +227,8 @@ class RssHelper:
}, },
} }
def parse(self, url, proxy: bool = False, timeout: Optional[int] = 15, headers: dict = None) -> Union[List[dict], None, bool]: def parse(self, url, proxy: bool = False,
timeout: Optional[int] = 15, headers: dict = None) -> Union[List[dict], None, bool]:
""" """
解析RSS订阅URL，获取RSS中的种子信息 解析RSS订阅URL，获取RSS中的种子信息
:param url: RSS地址 :param url: RSS地址
@@ -241,7 +241,7 @@ class RssHelper:
ret_array: list = [] ret_array: list = []
if not url: if not url:
return False return False
try: try:
ret = RequestUtils(proxies=settings.PROXY if proxy else None, ret = RequestUtils(proxies=settings.PROXY if proxy else None,
timeout=timeout, headers=headers).get_res(url) timeout=timeout, headers=headers).get_res(url)
@@ -250,7 +250,7 @@ class RssHelper:
except Exception as err: except Exception as err:
logger.error(f"获取RSS失败：{str(err)} - {traceback.format_exc()}") logger.error(f"获取RSS失败：{str(err)} - {traceback.format_exc()}")
return False return False
if ret: if ret:
ret_xml = None ret_xml = None
root = None root = None
@@ -258,9 +258,9 @@ class RssHelper:
# 检查响应大小，避免处理过大的RSS文件 # 检查响应大小，避免处理过大的RSS文件
raw_data = ret.content raw_data = ret.content
if raw_data and len(raw_data) > self.MAX_RSS_SIZE: if raw_data and len(raw_data) > self.MAX_RSS_SIZE:
logger.warning(f"RSS文件过大: {len(raw_data)/1024/1024:.1f}MB，跳过解析") logger.warning(f"RSS文件过大: {len(raw_data) / 1024 / 1024:.1f}MB，跳过解析")
return False return False
if raw_data: if raw_data:
try: try:
result = chardet.detect(raw_data) result = chardet.detect(raw_data)
@@ -279,7 +279,7 @@ class RssHelper:
ret.encoding = ret.apparent_encoding ret.encoding = ret.apparent_encoding
if not ret_xml: if not ret_xml:
ret_xml = ret.text ret_xml = ret.text
# 使用lxml.etree解析XML # 使用lxml.etree解析XML
parser = None parser = None
try: try:
@@ -307,42 +307,39 @@ class RssHelper:
finally: finally:
if parser is not None: if parser is not None:
del parser del parser
if root is None: if root is None:
logger.error("无法解析RSS内容") logger.error("无法解析RSS内容")
return False return False
# 查找所有item或entry节点 # 查找所有item或entry节点
items = root.xpath('.//item | .//entry') items = root.xpath('.//item | .//entry')
# 限制处理的条目数量 # 限制处理的条目数量
items_count = min(len(items), self.MAX_RSS_ITEMS) items_count = min(len(items), self.MAX_RSS_ITEMS)
if len(items) > self.MAX_RSS_ITEMS: if len(items) > self.MAX_RSS_ITEMS:
logger.warning(f"RSS条目过多: {len(items)}，仅处理前{self.MAX_RSS_ITEMS}个") logger.warning(f"RSS条目过多: {len(items)}，仅处理前{self.MAX_RSS_ITEMS}个")
for i, item in enumerate(items[:items_count]): for item in items[:items_count]:
try: try:
# 定期执行垃圾回收
if i > 0 and i % 100 == 0:
gc.collect()
# 使用xpath提取信息，更高效 # 使用xpath提取信息，更高效
title_nodes = item.xpath('.//title') title_nodes = item.xpath('.//title')
title = title_nodes[0].text if title_nodes and title_nodes[0].text else "" title = title_nodes[0].text if title_nodes and title_nodes[0].text else ""
if not title: if not title:
continue continue
# 描述 # 描述
desc_nodes = item.xpath('.//description | .//summary') desc_nodes = item.xpath('.//description | .//summary')
description = desc_nodes[0].text if desc_nodes and desc_nodes[0].text else "" description = desc_nodes[0].text if desc_nodes and desc_nodes[0].text else ""
# 种子页面 # 种子页面
link_nodes = item.xpath('.//link') link_nodes = item.xpath('.//link')
if link_nodes: if link_nodes:
link = link_nodes[0].text if hasattr(link_nodes[0], 'text') and link_nodes[0].text else link_nodes[0].get('href', '') link = link_nodes[0].text if hasattr(link_nodes[0], 'text') and link_nodes[0].text else \
link_nodes[0].get('href', '')
else: else:
link = "" link = ""
# 种子链接 # 种子链接
enclosure_nodes = item.xpath('.//enclosure') enclosure_nodes = item.xpath('.//enclosure')
enclosure = enclosure_nodes[0].get('url', '') if enclosure_nodes else "" enclosure = enclosure_nodes[0].get('url', '') if enclosure_nodes else ""
@@ -351,24 +348,24 @@ class RssHelper:
# 部分RSS只有link没有enclosure # 部分RSS只有link没有enclosure
if not enclosure and link: if not enclosure and link:
enclosure = link enclosure = link
# 大小 # 大小
size = 0 size = 0
if enclosure_nodes: if enclosure_nodes:
size_attr = enclosure_nodes[0].get('length', '0') size_attr = enclosure_nodes[0].get('length', '0')
if size_attr and str(size_attr).isdigit(): if size_attr and str(size_attr).isdigit():
size = int(size_attr) size = int(size_attr)
# 发布日期 # 发布日期
pubdate_nodes = item.xpath('.//pubDate | .//published | .//updated') pubdate_nodes = item.xpath('.//pubDate | .//published | .//updated')
pubdate = "" pubdate = ""
if pubdate_nodes and pubdate_nodes[0].text: if pubdate_nodes and pubdate_nodes[0].text:
pubdate = StringUtils.get_time(pubdate_nodes[0].text) pubdate = StringUtils.get_time(pubdate_nodes[0].text)
# 获取豆瓣昵称 # 获取豆瓣昵称
nickname_nodes = item.xpath('.//*[local-name()="creator"]') nickname_nodes = item.xpath('.//*[local-name()="creator"]')
nickname = nickname_nodes[0].text if nickname_nodes and nickname_nodes[0].text else "" nickname = nickname_nodes[0].text if nickname_nodes and nickname_nodes[0].text else ""
# 返回对象 # 返回对象
tmp_dict = { tmp_dict = {
'title': title, 'title': title,
@@ -382,11 +379,11 @@ class RssHelper:
if nickname: if nickname:
tmp_dict['nickname'] = nickname tmp_dict['nickname'] = nickname
ret_array.append(tmp_dict) ret_array.append(tmp_dict)
except Exception as e1: except Exception as e1:
logger.debug(f"解析RSS条目失败：{str(e1)} - {traceback.format_exc()}") logger.debug(f"解析RSS条目失败：{str(e1)} - {traceback.format_exc()}")
continue continue
except Exception as e2: except Exception as e2:
logger.error(f"解析RSS失败：{str(e2)} - {traceback.format_exc()}") logger.error(f"解析RSS失败：{str(e2)} - {traceback.format_exc()}")
# RSS过期检查 # RSS过期检查
@@ -403,8 +400,7 @@ class RssHelper:
del root del root
if ret_xml is not None: if ret_xml is not None:
del ret_xml del ret_xml
gc.collect()
return ret_array return ret_array
def get_rss_link(self, url: str, cookie: str, ua: str, proxy: bool = False) -> Tuple[str, str]: def get_rss_link(self, url: str, cookie: str, ua: str, proxy: bool = False) -> Tuple[str, str]:
@@ -446,7 +442,7 @@ class RssHelper:
return "", f"获取 {url} RSS链接失败，错误码：{res.status_code}，错误原因：{res.reason}" return "", f"获取 {url} RSS链接失败，错误码：{res.status_code}，错误原因：{res.reason}"
else: else:
return "", f"获取RSS链接失败：无法连接 {url} " return "", f"获取RSS链接失败：无法连接 {url} "
# 解析HTML # 解析HTML
if html_text: if html_text:
html = None html = None
@@ -459,7 +455,7 @@ class RssHelper:
finally: finally:
if html is not None: if html is not None:
del html del html
return "", f"获取RSS链接失败：{url}" return "", f"获取RSS链接失败：{url}"
except Exception as e: except Exception as e:
return "", f"获取 {url} RSS链接失败：{str(e)}" return "", f"获取 {url} RSS链接失败：{str(e)}"