GithubBackup/IP-Sentinel
1
0
Fork 0
mirror of https://github.com/hotyue/IP-Sentinel.git synced 2026-07-23 05:26:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(data): 兵工厂重构，升级 2025 高保真多平台 UA 轮换池；扩容活体新闻至 30 发纯双栈(IPv4+IPv6)满编矩阵

Browse Source
This commit is contained in:
hotyue
2026-05-23 03:23:49 +00:00
parent dad8e9477b
commit 866a884e44
3 changed files with 125 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
scripts/fetch_trends.py
View File

@@ -7,7 +7,6 @@ import time
import random
# ================== [路径防弹装甲] ==================
# 无论在哪里执行该脚本，都能精准反推项目根目录
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
PROJECT_ROOT = os.path.dirname(SCRIPT_DIR)
@@ -15,27 +14,15 @@ MAP_JSON_PATH = os.path.join(PROJECT_ROOT, "data", "map.json")
DATA_DIR = os.path.join(PROJECT_ROOT, "data", "keywords")
# ====================================================
# 特殊战区代码映射 (Google Trends RSS 要求)
GEO_FIX = {'UK': 'GB'}
FALLBACK_MAP = {'LA': 'US', 'MN': 'US', 'MO': 'HK'}
# ================== [核心修复 1: 兜底机制] ==================
# Google Trends 不支持的战区，降级抓取 US (美国) 通用流量
FALLBACK_MAP = {
'LA': 'US',
'MN': 'US'
,'MO': 'HK'
}
# ================== [核心修复 2: 随机 UA 池] ==================
USER_AGENTS = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2.1 Safari/605.1.15',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0'
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36'
]
def get_active_regions():
"""动态提取 map.json 中的战区 (适配 v3.5.0 大洲战区降维架构)"""
try:
with open(MAP_JSON_PATH, 'r', encoding='utf-8') as f:
data = json.load(f)
@@ -50,14 +37,9 @@ def get_active_regions():
return []
def fetch_trends(region_code):
"""从 Google Trends 抓取当日热搜"""
geo = GEO_FIX.get(region_code, region_code)
# 触发兜底判定
actual_geo = FALLBACK_MAP.get(geo, geo)
url = f"https://trends.google.com/trending/rss?geo={actual_geo}"
# 随机抽取 User-Agent
headers = {'User-Agent': random.choice(USER_AGENTS)}
try:
@@ -65,10 +47,7 @@ def fetch_trends(region_code):
with urllib.request.urlopen(req, timeout=10) as response:
xml_data = response.read()
root = ET.fromstring(xml_data)
# 如果触发了兜底，准备提示信息
fallback_msg = f" (兜底降级至 {actual_geo})" if actual_geo != geo else ""
words = [re.sub(r'[\n\r\t]', ' ', item.find('title').text).strip()
for item in root.findall('./channel/item')
if item.find('title') is not None]
@@ -78,7 +57,7 @@ def fetch_trends(region_code):
return [], ""
def update_file(region, new_words, fallback_msg=""):
"""滑动窗口更新，保留 200 条最热记录"""
"""滑动窗口更新，严格保留最新 100 条最热记录"""
os.makedirs(DATA_DIR, exist_ok=True)
file_path = os.path.join(DATA_DIR, f"kw_{region}.txt")
old_words = []
@@ -86,13 +65,13 @@ def update_file(region, new_words, fallback_msg=""):
with open(file_path, 'r', encoding='utf-8') as f:
old_words = [l.strip() for l in f if l.strip()]
# 新词排在最前面，去重
combined = new_words + [w for w in old_words if w not in new_words]
final_list = combined[:200]
# 【业务收敛】切片收紧至 100 条
final_list = combined[:100]
with open(file_path, 'w', encoding='utf-8') as f:
f.write('\n'.join(final_list) + '\n')
print(f"✅ [同步完成] {region}: 注入 {len(new_words)} 条新热点{fallback_msg}")
print(f"✅ [同步完成] {region}: 注入 {len(new_words)} 条新热点，维持总数 {len(final_list)} 条{fallback_msg}")
if __name__ == '__main__':
regions = get_active_regions()
@@ -106,8 +85,5 @@ if __name__ == '__main__':
words, fallback_msg = fetch_trends(r)
if words:
update_file(r, words, fallback_msg)
# [核心修复 3: 流量削峰] 随机休眠 1.5 到 3.5 秒
time.sleep(random.uniform(1.5, 3.5))
print("========== 热词抓取引擎执行完毕 ==========")