remove gc

This commit is contained in:
jxxghp
2025-06-08 21:19:15 +08:00
parent 7f8e50f83d
commit 0a20234268
2 changed files with 30 additions and 38 deletions
+2 -6
View File
@@ -1,8 +1,8 @@
from typing import Callable, Any, Optional from typing import Callable, Any, Optional
import gc
from playwright.sync_api import sync_playwright, Page
from cf_clearance import sync_cf_retry, sync_stealth from cf_clearance import sync_cf_retry, sync_stealth
from playwright.sync_api import sync_playwright, Page
from app.log import logger from app.log import logger
@@ -67,8 +67,6 @@ class PlaywrightHelper:
context.close() context.close()
if browser: if browser:
browser.close() browser.close()
# 强制垃圾回收
gc.collect()
except Exception as e: except Exception as e:
logger.error(f"Playwright初始化失败: {str(e)}") logger.error(f"Playwright初始化失败: {str(e)}")
@@ -120,8 +118,6 @@ class PlaywrightHelper:
context.close() context.close()
if browser: if browser:
browser.close() browser.close()
# 强制垃圾回收
gc.collect()
except Exception as e: except Exception as e:
logger.error(f"Playwright初始化失败: {str(e)}") logger.error(f"Playwright初始化失败: {str(e)}")
+6 -10
View File
@@ -1,4 +1,3 @@
import gc
import re import re
import traceback import traceback
from typing import List, Tuple, Union, Optional from typing import List, Tuple, Union, Optional
@@ -228,7 +227,8 @@ class RssHelper:
}, },
} }
def parse(self, url, proxy: bool = False, timeout: Optional[int] = 15, headers: dict = None) -> Union[List[dict], None, bool]: def parse(self, url, proxy: bool = False,
timeout: Optional[int] = 15, headers: dict = None) -> Union[List[dict], None, bool]:
""" """
解析RSS订阅URL,获取RSS中的种子信息 解析RSS订阅URL,获取RSS中的种子信息
:param url: RSS地址 :param url: RSS地址
@@ -258,7 +258,7 @@ class RssHelper:
# 检查响应大小,避免处理过大的RSS文件 # 检查响应大小,避免处理过大的RSS文件
raw_data = ret.content raw_data = ret.content
if raw_data and len(raw_data) > self.MAX_RSS_SIZE: if raw_data and len(raw_data) > self.MAX_RSS_SIZE:
logger.warning(f"RSS文件过大: {len(raw_data)/1024/1024:.1f}MB,跳过解析") logger.warning(f"RSS文件过大: {len(raw_data) / 1024 / 1024:.1f}MB,跳过解析")
return False return False
if raw_data: if raw_data:
@@ -320,12 +320,8 @@ class RssHelper:
if len(items) > self.MAX_RSS_ITEMS: if len(items) > self.MAX_RSS_ITEMS:
logger.warning(f"RSS条目过多: {len(items)},仅处理前{self.MAX_RSS_ITEMS}") logger.warning(f"RSS条目过多: {len(items)},仅处理前{self.MAX_RSS_ITEMS}")
for i, item in enumerate(items[:items_count]): for item in items[:items_count]:
try: try:
# 定期执行垃圾回收
if i > 0 and i % 100 == 0:
gc.collect()
# 使用xpath提取信息,更高效 # 使用xpath提取信息,更高效
title_nodes = item.xpath('.//title') title_nodes = item.xpath('.//title')
title = title_nodes[0].text if title_nodes and title_nodes[0].text else "" title = title_nodes[0].text if title_nodes and title_nodes[0].text else ""
@@ -339,7 +335,8 @@ class RssHelper:
# 种子页面 # 种子页面
link_nodes = item.xpath('.//link') link_nodes = item.xpath('.//link')
if link_nodes: if link_nodes:
link = link_nodes[0].text if hasattr(link_nodes[0], 'text') and link_nodes[0].text else link_nodes[0].get('href', '') link = link_nodes[0].text if hasattr(link_nodes[0], 'text') and link_nodes[0].text else \
link_nodes[0].get('href', '')
else: else:
link = "" link = ""
@@ -403,7 +400,6 @@ class RssHelper:
del root del root
if ret_xml is not None: if ret_xml is not None:
del ret_xml del ret_xml
gc.collect()
return ret_array return ret_array