mirror of
https://github.com/mskatoni/ni-mail.git
synced 2026-09-05 23:47:36 +08:00
feat: complete ni-mail rewrite with Graph API, IMAP, OTP extractor, Web UI and Docker deployment
This commit is contained in:
@@ -0,0 +1,422 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
# from app.errors import build_error_payload
|
||||
from app.services.http import get_response_details
|
||||
|
||||
# Token 端点
|
||||
TOKEN_URL_TEMPLATE = "https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token"
|
||||
TOKEN_URL_GRAPH = TOKEN_URL_TEMPLATE.format(tenant="common")
|
||||
DEFAULT_GRAPH_SCOPE = "https://graph.microsoft.com/.default"
|
||||
GRAPH_MAIL_READ_SCOPES = ("Mail.Read", "Mail.ReadWrite")
|
||||
|
||||
# Graph API 返回 401 时表示账号授权失效(与 token endpoint 失败不同)
|
||||
GRAPH_AUTH_EXPIRED_STATUS = 401
|
||||
|
||||
|
||||
def build_proxies(proxy_url: str) -> Optional[Dict[str, str]]:
|
||||
"""构建 requests 的 proxies 参数"""
|
||||
if not proxy_url:
|
||||
return None
|
||||
return {"http": proxy_url, "https": proxy_url}
|
||||
|
||||
|
||||
def build_token_url(tenant: str | None = None) -> str:
|
||||
"""按 tenant 生成 Microsoft OAuth token endpoint。"""
|
||||
normalized_tenant = (tenant or "common").strip() or "common"
|
||||
return TOKEN_URL_TEMPLATE.format(tenant=normalized_tenant)
|
||||
|
||||
|
||||
def get_access_token_graph_result(client_id: str, refresh_token: str, proxy_url: str = None) -> Dict[str, Any]:
|
||||
"""获取 Graph API access_token(包含错误详情)"""
|
||||
try:
|
||||
proxies = build_proxies(proxy_url)
|
||||
res = requests.post(
|
||||
TOKEN_URL_GRAPH,
|
||||
data={
|
||||
"client_id": client_id,
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": refresh_token,
|
||||
"scope": DEFAULT_GRAPH_SCOPE,
|
||||
},
|
||||
timeout=30,
|
||||
proxies=proxies,
|
||||
)
|
||||
|
||||
if res.status_code != 200:
|
||||
details = get_response_details(res)
|
||||
return {
|
||||
"success": False,
|
||||
"error": build_error_payload(
|
||||
"GRAPH_TOKEN_FAILED",
|
||||
"获取访问令牌失败",
|
||||
"GraphAPIError",
|
||||
res.status_code,
|
||||
details,
|
||||
),
|
||||
}
|
||||
|
||||
payload = res.json()
|
||||
access_token = payload.get("access_token")
|
||||
if not access_token:
|
||||
return {
|
||||
"success": False,
|
||||
"error": build_error_payload(
|
||||
"GRAPH_TOKEN_MISSING",
|
||||
"获取访问令牌失败",
|
||||
"GraphAPIError",
|
||||
res.status_code,
|
||||
payload,
|
||||
),
|
||||
}
|
||||
|
||||
# 根据 Microsoft Learn 文档:refresh token 可能会在每次使用时"自我替换",应保存新的 refresh_token(如有)。
|
||||
new_refresh_token = payload.get("refresh_token")
|
||||
return {
|
||||
"success": True,
|
||||
"access_token": access_token,
|
||||
"refresh_token": new_refresh_token,
|
||||
"new_refresh_token": new_refresh_token,
|
||||
"scope": payload.get("scope", ""),
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"success": False,
|
||||
"error": build_error_payload(
|
||||
"GRAPH_TOKEN_EXCEPTION",
|
||||
"获取访问令牌失败",
|
||||
type(exc).__name__,
|
||||
500,
|
||||
str(exc),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def has_mail_read_permission(scope: Any) -> bool:
|
||||
scope_str = str(scope or "")
|
||||
return any(mail_scope in scope_str for mail_scope in GRAPH_MAIL_READ_SCOPES)
|
||||
|
||||
|
||||
def get_access_token_graph(client_id: str, refresh_token: str, proxy_url: str = None) -> Optional[str]:
|
||||
"""获取 Graph API access_token"""
|
||||
result = get_access_token_graph_result(client_id, refresh_token, proxy_url)
|
||||
if result.get("success"):
|
||||
return result.get("access_token")
|
||||
return None
|
||||
|
||||
|
||||
def get_emails_graph(
|
||||
client_id: str,
|
||||
refresh_token: str,
|
||||
folder: str = "inbox",
|
||||
skip: int = 0,
|
||||
top: int = 20,
|
||||
proxy_url: str = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""使用 Graph API 获取邮件列表(支持分页和文件夹选择)"""
|
||||
token_result = get_access_token_graph_result(client_id, refresh_token, proxy_url)
|
||||
if not token_result.get("success"):
|
||||
return {"success": False, "error": token_result.get("error")}
|
||||
|
||||
access_token = token_result.get("access_token")
|
||||
scope = token_result.get("scope", "")
|
||||
if not has_mail_read_permission(scope):
|
||||
return {
|
||||
"success": False,
|
||||
"auth_expired": True,
|
||||
"no_mail_permission": True,
|
||||
"error": build_error_payload(
|
||||
"NO_MAIL_PERMISSION",
|
||||
"此账号未授予邮件读取权限 (scope 中不含 Mail.Read)",
|
||||
"PermissionError",
|
||||
403,
|
||||
f"scope={scope}",
|
||||
),
|
||||
}
|
||||
|
||||
try:
|
||||
folder_map = {
|
||||
"inbox": "inbox",
|
||||
"junkemail": "junkemail",
|
||||
"deleteditems": "deleteditems",
|
||||
"trash": "deleteditems",
|
||||
}
|
||||
folder_name = folder_map.get((folder or "").lower(), "inbox")
|
||||
|
||||
url = f"https://graph.microsoft.com/v1.0/me/mailFolders/{folder_name}/messages"
|
||||
params = {
|
||||
"$top": top,
|
||||
"$skip": skip,
|
||||
"$select": "id,subject,from,receivedDateTime,isRead,hasAttachments,bodyPreview",
|
||||
"$orderby": "receivedDateTime desc",
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {access_token}",
|
||||
"Prefer": "outlook.body-content-type='text'",
|
||||
}
|
||||
|
||||
proxies = build_proxies(proxy_url)
|
||||
res = requests.get(url, headers=headers, params=params, timeout=30, proxies=proxies)
|
||||
|
||||
if res.status_code != 200:
|
||||
details = get_response_details(res)
|
||||
return {
|
||||
"success": False,
|
||||
"auth_expired": res.status_code == GRAPH_AUTH_EXPIRED_STATUS,
|
||||
"error": build_error_payload(
|
||||
"EMAIL_FETCH_FAILED",
|
||||
"获取邮件失败,请检查账号配置",
|
||||
"GraphAPIError",
|
||||
res.status_code,
|
||||
details,
|
||||
),
|
||||
}
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"emails": res.json().get("value", []),
|
||||
"new_refresh_token": token_result.get("refresh_token"),
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"success": False,
|
||||
"error": build_error_payload(
|
||||
"EMAIL_FETCH_FAILED",
|
||||
"获取邮件失败,请检查账号配置",
|
||||
type(exc).__name__,
|
||||
500,
|
||||
str(exc),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def get_email_detail_graph(
|
||||
client_id: str,
|
||||
refresh_token: str,
|
||||
message_id: str,
|
||||
proxy_url: str = None,
|
||||
) -> Optional[Dict]:
|
||||
"""使用 Graph API 获取邮件详情"""
|
||||
access_token = get_access_token_graph(client_id, refresh_token, proxy_url)
|
||||
if not access_token:
|
||||
return None
|
||||
|
||||
try:
|
||||
url = f"https://graph.microsoft.com/v1.0/me/messages/{message_id}"
|
||||
params = {
|
||||
"$select": "id,subject,from,toRecipients,ccRecipients,receivedDateTime,isRead,hasAttachments,body,bodyPreview"
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {access_token}",
|
||||
"Prefer": "outlook.body-content-type='html'",
|
||||
}
|
||||
|
||||
proxies = build_proxies(proxy_url)
|
||||
res = requests.get(url, headers=headers, params=params, timeout=30, proxies=proxies)
|
||||
|
||||
if res.status_code != 200:
|
||||
return None
|
||||
|
||||
return res.json()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def get_email_raw_graph(
|
||||
client_id: str,
|
||||
refresh_token: str,
|
||||
message_id: str,
|
||||
proxy_url: str = None,
|
||||
) -> Optional[str]:
|
||||
"""使用 Graph API 获取邮件 MIME RAW 内容。"""
|
||||
access_token = get_access_token_graph(client_id, refresh_token, proxy_url)
|
||||
if not access_token:
|
||||
return None
|
||||
|
||||
try:
|
||||
url = f"https://graph.microsoft.com/v1.0/me/messages/{message_id}/$value"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {access_token}",
|
||||
}
|
||||
|
||||
proxies = build_proxies(proxy_url)
|
||||
res = requests.get(url, headers=headers, timeout=30, proxies=proxies)
|
||||
|
||||
if res.status_code != 200:
|
||||
return None
|
||||
|
||||
res.encoding = res.encoding or "utf-8"
|
||||
return res.text
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def test_refresh_token(client_id: str, refresh_token: str, proxy_url: str = None) -> tuple[bool, str | None]:
|
||||
"""测试 refresh token 是否有效,返回 (是否成功, 错误信息)"""
|
||||
ok, err, _new_refresh_token = test_refresh_token_with_rotation(client_id, refresh_token, proxy_url)
|
||||
return ok, err
|
||||
|
||||
|
||||
def test_refresh_token_with_rotation(
|
||||
client_id: str,
|
||||
refresh_token: str,
|
||||
proxy_url: str = None,
|
||||
*,
|
||||
tenant: str = "common",
|
||||
scope: str = DEFAULT_GRAPH_SCOPE,
|
||||
max_retries: int = 3,
|
||||
) -> tuple[bool, str | None, str | None]:
|
||||
"""测试 refresh token 是否有效;如服务端返回新的 refresh_token,则一并返回(用于滚动更新)。
|
||||
支持指数退避重试,遇到 429 时优先读取 Retry-After 头。"""
|
||||
import time
|
||||
|
||||
proxies = build_proxies(proxy_url)
|
||||
resolved_scope = (scope or DEFAULT_GRAPH_SCOPE).strip() or DEFAULT_GRAPH_SCOPE
|
||||
url = build_token_url(tenant)
|
||||
data = {
|
||||
"client_id": client_id,
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": refresh_token,
|
||||
"scope": resolved_scope,
|
||||
}
|
||||
|
||||
last_error_msg = None
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
res = requests.post(url, data=data, timeout=15, proxies=proxies)
|
||||
|
||||
if res.status_code == 200:
|
||||
try:
|
||||
payload = res.json()
|
||||
except Exception:
|
||||
payload = {}
|
||||
new_refresh_token = payload.get("refresh_token")
|
||||
return True, None, new_refresh_token
|
||||
|
||||
# 429 限流:读取 Retry-After 并退避
|
||||
if res.status_code == 429:
|
||||
retry_after = None
|
||||
try:
|
||||
retry_after = int(res.headers.get("Retry-After", 0))
|
||||
except Exception:
|
||||
retry_after = None
|
||||
wait = retry_after if retry_after else (2**attempt)
|
||||
last_error_msg = f"请求被限流 (429),{wait}s 后重试"
|
||||
if attempt < max_retries:
|
||||
time.sleep(wait)
|
||||
continue
|
||||
|
||||
try:
|
||||
error_data = res.json()
|
||||
except Exception:
|
||||
error_data = {}
|
||||
error_msg = None
|
||||
if isinstance(error_data, dict):
|
||||
error_msg = error_data.get("error_description") or error_data.get("error")
|
||||
if not error_msg:
|
||||
details = get_response_details(res)
|
||||
error_msg = str(details)[:800] if details is not None else "未知错误"
|
||||
last_error_msg = str(error_msg)
|
||||
# 非 429 的明确错误响应(如 400/401/403)不需要重试,直接返回
|
||||
return False, last_error_msg, None
|
||||
except Exception as e:
|
||||
last_error_msg = f"请求异常: {str(e)}"
|
||||
if attempt < max_retries:
|
||||
time.sleep(2**attempt)
|
||||
continue
|
||||
return False, last_error_msg, None
|
||||
|
||||
return False, last_error_msg or "请求失败", None
|
||||
|
||||
|
||||
def delete_emails_graph(
|
||||
client_id: str,
|
||||
refresh_token: str,
|
||||
message_ids: List[str],
|
||||
proxy_url: str = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""通过 Graph API 批量删除邮件(永久删除)"""
|
||||
token_result = get_access_token_graph_result(client_id, refresh_token, proxy_url)
|
||||
if not token_result.get("success"):
|
||||
return {"success": False, "error": token_result.get("error")}
|
||||
|
||||
access_token = token_result.get("access_token")
|
||||
if not access_token:
|
||||
return {
|
||||
"success": False,
|
||||
"error": build_error_payload(
|
||||
"GRAPH_TOKEN_FAILED",
|
||||
"获取访问令牌失败",
|
||||
"GraphAPIError",
|
||||
500,
|
||||
"empty_access_token",
|
||||
),
|
||||
}
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {access_token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
# Graph API batch 请求每次最多 20
|
||||
batch_size = 20
|
||||
success_count = 0
|
||||
failed_count = 0
|
||||
errors: List[str] = []
|
||||
|
||||
for i in range(0, len(message_ids), batch_size):
|
||||
batch = message_ids[i : i + batch_size]
|
||||
|
||||
batch_requests = []
|
||||
for idx, msg_id in enumerate(batch):
|
||||
batch_requests.append({"id": str(idx), "method": "DELETE", "url": f"/me/messages/{msg_id}"})
|
||||
|
||||
try:
|
||||
proxies = build_proxies(proxy_url)
|
||||
response = requests.post(
|
||||
"https://graph.microsoft.com/v1.0/$batch",
|
||||
headers=headers,
|
||||
json={"requests": batch_requests},
|
||||
timeout=30,
|
||||
proxies=proxies,
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
results = response.json().get("responses", [])
|
||||
for res in results:
|
||||
if res.get("status") in [200, 204]:
|
||||
success_count += 1
|
||||
else:
|
||||
failed_count += 1
|
||||
try:
|
||||
errors.append(f"Msg ID: {batch[int(res['id'])]}, Status: {res.get('status')}")
|
||||
except Exception:
|
||||
errors.append(f"Status: {res.get('status')}")
|
||||
else:
|
||||
failed_count += len(batch)
|
||||
errors.append(f"Batch request failed: {response.text}")
|
||||
except Exception as e:
|
||||
failed_count += len(batch)
|
||||
errors.append(f"Network error: {str(e)}")
|
||||
|
||||
result: Dict[str, Any] = {
|
||||
"success": success_count > 0,
|
||||
"partial_success": success_count > 0 and failed_count > 0,
|
||||
"success_count": success_count,
|
||||
"failed_count": failed_count,
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
if not result["success"]:
|
||||
result["error"] = build_error_payload(
|
||||
"EMAIL_DELETE_FAILED",
|
||||
"删除邮件失败",
|
||||
"GraphAPIError",
|
||||
502,
|
||||
{"failed_count": failed_count, "errors": errors[:10]},
|
||||
)
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
def get_response_details(response: requests.Response) -> Any:
|
||||
try:
|
||||
return response.json()
|
||||
except Exception:
|
||||
return response.text or response.reason
|
||||
@@ -0,0 +1,780 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import email
|
||||
import hashlib
|
||||
import imaplib
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from email.header import decode_header
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
# from app.errors import build_error_payload
|
||||
from outlook_web.services.graph import get_access_token_graph
|
||||
from app.services.http import get_response_details
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
# Token 端点
|
||||
TOKEN_URL_IMAP = "https://login.microsoftonline.com/consumers/oauth2/v2.0/token"
|
||||
|
||||
# IMAP 服务器配置
|
||||
IMAP_SERVER_NEW = "outlook.live.com"
|
||||
IMAP_PORT = 993
|
||||
|
||||
_token_cache: Dict[str, tuple] = {}
|
||||
_token_cache_lock = threading.Lock()
|
||||
|
||||
|
||||
def decode_header_value(header_value: str) -> str:
|
||||
"""解码邮件头字段"""
|
||||
if not header_value:
|
||||
return ""
|
||||
try:
|
||||
decoded_parts = decode_header(str(header_value))
|
||||
decoded_string = ""
|
||||
for part, charset in decoded_parts:
|
||||
if isinstance(part, bytes):
|
||||
try:
|
||||
decoded_string += part.decode(charset if charset else "utf-8", "replace")
|
||||
except (LookupError, UnicodeDecodeError):
|
||||
decoded_string += part.decode("utf-8", "replace")
|
||||
else:
|
||||
decoded_string += str(part)
|
||||
return decoded_string
|
||||
except Exception:
|
||||
return str(header_value) if header_value else ""
|
||||
|
||||
|
||||
def get_email_body(msg) -> str:
|
||||
"""提取邮件正文
|
||||
|
||||
优先返回 text/plain,但如果内容太短(<20字符),则回退到 text/html。
|
||||
这解决了 Figma 等服务的邮件问题:它们的 text/plain 部分可能几乎为空,
|
||||
而正文内容都在 text/html 部分。
|
||||
"""
|
||||
plain_text = ""
|
||||
html_text = ""
|
||||
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
content_type = part.get_content_type()
|
||||
content_disposition = str(part.get("Content-Disposition", ""))
|
||||
|
||||
if "attachment" in content_disposition:
|
||||
continue
|
||||
|
||||
if content_type == "text/plain" and not plain_text:
|
||||
try:
|
||||
payload = part.get_payload(decode=True)
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
plain_text = payload.decode(charset, errors="replace")
|
||||
except Exception:
|
||||
continue
|
||||
elif content_type == "text/html" and not html_text:
|
||||
try:
|
||||
payload = part.get_payload(decode=True)
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
html_text = payload.decode(charset, errors="replace")
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if plain_text and html_text:
|
||||
break
|
||||
else:
|
||||
try:
|
||||
payload = msg.get_payload(decode=True)
|
||||
charset = msg.get_content_charset() or "utf-8"
|
||||
content = payload.decode(charset, errors="replace")
|
||||
if msg.get_content_type() == "text/html":
|
||||
html_text = content
|
||||
else:
|
||||
plain_text = content
|
||||
except Exception:
|
||||
plain_text = str(msg.get_payload())
|
||||
|
||||
# 如果 text/plain 太短(<20字符),回退到 text/html
|
||||
# 这解决了 Figma 等服务的邮件问题
|
||||
if len(plain_text.strip()) >= 20:
|
||||
return plain_text
|
||||
return html_text or plain_text
|
||||
|
||||
|
||||
def get_email_body_and_type(msg) -> tuple:
|
||||
"""提取邮件正文和类型(用于需要区分 HTML/Text 的场景)
|
||||
|
||||
返回 (body, body_type) 元组:
|
||||
- body: 邮件正文内容
|
||||
- body_type: "html" 或 "text"
|
||||
"""
|
||||
plain_text = ""
|
||||
html_text = ""
|
||||
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
content_type = part.get_content_type()
|
||||
content_disposition = str(part.get("Content-Disposition", ""))
|
||||
|
||||
if "attachment" in content_disposition:
|
||||
continue
|
||||
|
||||
if content_type == "text/plain" and not plain_text:
|
||||
try:
|
||||
payload = part.get_payload(decode=True)
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
plain_text = payload.decode(charset, errors="replace")
|
||||
except Exception:
|
||||
continue
|
||||
elif content_type == "text/html" and not html_text:
|
||||
try:
|
||||
payload = part.get_payload(decode=True)
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
html_text = payload.decode(charset, errors="replace")
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if plain_text and html_text:
|
||||
break
|
||||
else:
|
||||
try:
|
||||
payload = msg.get_payload(decode=True)
|
||||
charset = msg.get_content_charset() or "utf-8"
|
||||
content = payload.decode(charset, errors="replace")
|
||||
if msg.get_content_type() == "text/html":
|
||||
html_text = content
|
||||
else:
|
||||
plain_text = content
|
||||
except Exception:
|
||||
plain_text = str(msg.get_payload())
|
||||
|
||||
# 如果 text/plain 太短(<20字符),回退到 text/html
|
||||
if len(plain_text.strip()) >= 20:
|
||||
return plain_text, "text"
|
||||
return html_text or plain_text, "html" if html_text else "text"
|
||||
|
||||
|
||||
def _select_folder(connection, folder: str) -> Optional[str]:
|
||||
folder_map = {
|
||||
"inbox": ["INBOX"],
|
||||
"junk": ["Junk", "Junk Email", "Spam", "垃圾邮件"],
|
||||
"junkemail": ["Junk", "Junk Email", "Spam", "垃圾邮件"],
|
||||
"deleteditems": ["Deleted", "Deleted Items", "Trash", "已删除邮件"],
|
||||
"trash": ["Deleted", "Deleted Items", "Trash", "已删除邮件"],
|
||||
}
|
||||
candidates = folder_map.get((folder or "").lower(), [folder or "INBOX"])
|
||||
for candidate in candidates:
|
||||
for select_target in (f'"{candidate}"', candidate):
|
||||
try:
|
||||
status, _ = connection.select(select_target, readonly=True)
|
||||
if status == "OK":
|
||||
return candidate
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _get_html_body(msg) -> str:
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
if part.get_content_type() == "text/html":
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload:
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
return payload.decode(charset, errors="replace")
|
||||
else:
|
||||
if msg.get_content_type() == "text/html":
|
||||
payload = msg.get_payload(decode=True)
|
||||
if payload:
|
||||
return payload.decode(msg.get_content_charset() or "utf-8", errors="replace")
|
||||
return ""
|
||||
|
||||
|
||||
def _parse_batch_fetch_response(all_data: list) -> List[tuple]:
|
||||
results = []
|
||||
for item in all_data:
|
||||
header = None
|
||||
raw_email = None
|
||||
|
||||
if isinstance(item, tuple) and len(item) == 2:
|
||||
first, second = item
|
||||
if isinstance(first, (bytes, bytearray)) and isinstance(second, (bytes, bytearray)):
|
||||
header = bytes(first)
|
||||
raw_email = bytes(second)
|
||||
elif isinstance(first, tuple) and len(first) == 2:
|
||||
nested_header, nested_raw = first
|
||||
if isinstance(nested_header, (bytes, bytearray)) and isinstance(nested_raw, (bytes, bytearray)):
|
||||
header = bytes(nested_header)
|
||||
raw_email = bytes(nested_raw)
|
||||
|
||||
if not isinstance(header, (bytes, bytearray)) or not isinstance(raw_email, (bytes, bytearray)):
|
||||
continue
|
||||
|
||||
msg_id_str = header.split(b" ", 1)[0].decode("ascii", errors="ignore").strip()
|
||||
if not msg_id_str:
|
||||
continue
|
||||
results.append((msg_id_str, raw_email))
|
||||
return results
|
||||
|
||||
|
||||
def _make_cache_key(client_id: str, refresh_token: str) -> str:
|
||||
rt_hash = hashlib.sha256(refresh_token.encode("utf-8")).hexdigest()[:16]
|
||||
return f"{client_id}:{rt_hash}"
|
||||
|
||||
|
||||
def clear_imap_token_cache(client_id: str = None) -> None:
|
||||
with _token_cache_lock:
|
||||
if client_id is None:
|
||||
_token_cache.clear()
|
||||
else:
|
||||
keys_to_remove = [k for k in _token_cache if k.startswith(f"{client_id}:")]
|
||||
for key in keys_to_remove:
|
||||
del _token_cache[key]
|
||||
|
||||
|
||||
def get_access_token_imap_result(client_id: str, refresh_token: str) -> Dict[str, Any]:
|
||||
"""获取 IMAP access_token(包含错误详情)"""
|
||||
cache_key = _make_cache_key(client_id, refresh_token)
|
||||
with _token_cache_lock:
|
||||
cached = _token_cache.get(cache_key)
|
||||
if cached:
|
||||
access_token, expires_at = cached
|
||||
if time.monotonic() < expires_at:
|
||||
return {"success": True, "access_token": access_token}
|
||||
|
||||
try:
|
||||
res = requests.post(
|
||||
TOKEN_URL_IMAP,
|
||||
data={
|
||||
"client_id": client_id,
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": refresh_token,
|
||||
"scope": "https://outlook.office.com/IMAP.AccessAsUser.All offline_access",
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
if res.status_code != 200:
|
||||
details = get_response_details(res)
|
||||
return {
|
||||
"success": False,
|
||||
"error": build_error_payload(
|
||||
"IMAP_TOKEN_FAILED",
|
||||
"获取访问令牌失败",
|
||||
"IMAPError",
|
||||
res.status_code,
|
||||
details,
|
||||
),
|
||||
}
|
||||
|
||||
payload = res.json()
|
||||
access_token = payload.get("access_token")
|
||||
if not access_token:
|
||||
return {
|
||||
"success": False,
|
||||
"error": build_error_payload(
|
||||
"IMAP_TOKEN_MISSING",
|
||||
"获取访问令牌失败",
|
||||
"IMAPError",
|
||||
res.status_code,
|
||||
payload,
|
||||
),
|
||||
}
|
||||
|
||||
expires_in = int(payload.get("expires_in", 3599))
|
||||
ttl = max(0, expires_in - 60)
|
||||
with _token_cache_lock:
|
||||
_token_cache[cache_key] = (access_token, time.monotonic() + ttl)
|
||||
|
||||
return {"success": True, "access_token": access_token}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"success": False,
|
||||
"error": build_error_payload(
|
||||
"IMAP_TOKEN_EXCEPTION",
|
||||
"获取访问令牌失败",
|
||||
type(exc).__name__,
|
||||
500,
|
||||
str(exc),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def get_access_token_imap(client_id: str, refresh_token: str) -> Optional[str]:
|
||||
"""获取 IMAP access_token"""
|
||||
result = get_access_token_imap_result(client_id, refresh_token)
|
||||
if result.get("success"):
|
||||
return result.get("access_token")
|
||||
return None
|
||||
|
||||
|
||||
def get_emails_imap(
|
||||
account: str,
|
||||
client_id: str,
|
||||
refresh_token: str,
|
||||
folder: str = "inbox",
|
||||
skip: int = 0,
|
||||
top: int = 20,
|
||||
) -> Dict[str, Any]:
|
||||
"""使用 IMAP 获取邮件列表(支持分页和文件夹选择)- 默认使用新版服务器"""
|
||||
return get_emails_imap_with_server(account, client_id, refresh_token, folder, skip, top, IMAP_SERVER_NEW)
|
||||
|
||||
|
||||
def get_emails_imap_with_server(
|
||||
account: str,
|
||||
client_id: str,
|
||||
refresh_token: str,
|
||||
folder: str = "inbox",
|
||||
skip: int = 0,
|
||||
top: int = 20,
|
||||
server: str = IMAP_SERVER_NEW,
|
||||
) -> Dict[str, Any]:
|
||||
"""使用 IMAP 获取邮件列表(支持分页、文件夹选择和服务器选择)"""
|
||||
token_result = get_access_token_imap_result(client_id, refresh_token)
|
||||
if not token_result.get("success"):
|
||||
return {"success": False, "error": token_result.get("error")}
|
||||
|
||||
access_token = token_result.get("access_token")
|
||||
|
||||
connection = None
|
||||
try:
|
||||
connection = imaplib.IMAP4_SSL(server, IMAP_PORT)
|
||||
auth_string = f"user={account}\1auth=Bearer {access_token}\1\1".encode("utf-8")
|
||||
connection.authenticate("XOAUTH2", lambda x: auth_string)
|
||||
|
||||
selected_folder = _select_folder(connection, folder)
|
||||
|
||||
if not selected_folder:
|
||||
try:
|
||||
status, folder_list = connection.list()
|
||||
available_folders = []
|
||||
if status == "OK" and folder_list:
|
||||
for folder_item in folder_list:
|
||||
if isinstance(folder_item, bytes):
|
||||
available_folders.append(folder_item.decode("utf-8", errors="ignore"))
|
||||
else:
|
||||
available_folders.append(str(folder_item))
|
||||
|
||||
error_details = {
|
||||
"last_error": "select folder failed",
|
||||
"tried_folder": folder,
|
||||
"available_folders": available_folders[:10],
|
||||
}
|
||||
except Exception:
|
||||
error_details = {
|
||||
"last_error": "select folder failed",
|
||||
"tried_folder": folder,
|
||||
}
|
||||
|
||||
return {
|
||||
"success": False,
|
||||
"error": build_error_payload(
|
||||
"EMAIL_FETCH_FAILED",
|
||||
"无法访问文件夹,请检查账号配置",
|
||||
"IMAPSelectError",
|
||||
500,
|
||||
error_details,
|
||||
),
|
||||
}
|
||||
|
||||
status, messages = connection.search(None, "ALL")
|
||||
if status != "OK":
|
||||
_LOGGER.debug(
|
||||
"[PERF] imap_search | account=%s | server=%s | folder=%s | status=%s (非OK)",
|
||||
account,
|
||||
server,
|
||||
selected_folder,
|
||||
status,
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
"error": build_error_payload(
|
||||
"EMAIL_FETCH_FAILED",
|
||||
"获取邮件失败,请检查账号配置",
|
||||
"IMAPSearchError",
|
||||
500,
|
||||
f"search status={status}",
|
||||
),
|
||||
}
|
||||
if not messages or not messages[0]:
|
||||
_LOGGER.debug(
|
||||
"[PERF] imap_search | account=%s | server=%s | folder=%s | total=0 (空信箱)",
|
||||
account,
|
||||
server,
|
||||
selected_folder,
|
||||
)
|
||||
return {"success": True, "emails": []}
|
||||
|
||||
message_ids = messages[0].split()
|
||||
total = len(message_ids)
|
||||
start_idx = max(0, total - skip - top)
|
||||
end_idx = total - skip
|
||||
|
||||
_LOGGER.debug(
|
||||
"[PERF] imap_search | account=%s | server=%s | folder=%s | total=%d | skip=%d | top=%d | slice=[%d:%d]",
|
||||
account,
|
||||
server,
|
||||
selected_folder,
|
||||
total,
|
||||
skip,
|
||||
top,
|
||||
start_idx,
|
||||
end_idx,
|
||||
)
|
||||
|
||||
if start_idx >= end_idx:
|
||||
return {"success": True, "emails": []}
|
||||
|
||||
paged_ids = message_ids[start_idx:end_idx][::-1]
|
||||
emails_data = []
|
||||
|
||||
ids_str = b",".join(paged_ids)
|
||||
status, all_data = connection.fetch(ids_str, "(RFC822)")
|
||||
if status != "OK":
|
||||
_LOGGER.debug(
|
||||
"[PERF] imap_fetch | account=%s | batch fetch失败 status=%s",
|
||||
account,
|
||||
status,
|
||||
)
|
||||
return {"success": True, "emails": emails_data}
|
||||
|
||||
for msg_id_str, raw_email in _parse_batch_fetch_response(all_data or []):
|
||||
try:
|
||||
msg = email.message_from_bytes(raw_email)
|
||||
body_preview = get_email_body(msg)
|
||||
emails_data.append(
|
||||
{
|
||||
"id": msg_id_str,
|
||||
"subject": decode_header_value(msg.get("Subject", "无主题")),
|
||||
"from": decode_header_value(msg.get("From", "未知发件人")),
|
||||
"date": msg.get("Date", "未知时间"),
|
||||
"body_preview": (body_preview[:200] + "..." if len(body_preview) > 200 else body_preview),
|
||||
}
|
||||
)
|
||||
except Exception as fetch_err:
|
||||
_LOGGER.debug(
|
||||
"[PERF] imap_fetch | account=%s | msg_id=%s | 解析失败: %s",
|
||||
account,
|
||||
msg_id_str,
|
||||
fetch_err,
|
||||
)
|
||||
continue
|
||||
|
||||
_LOGGER.debug(
|
||||
"[PERF] imap_result | account=%s | server=%s | fetched=%d / requested=%d",
|
||||
account,
|
||||
server,
|
||||
len(emails_data),
|
||||
len(paged_ids),
|
||||
)
|
||||
return {"success": True, "emails": emails_data}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"success": False,
|
||||
"error": build_error_payload(
|
||||
"EMAIL_FETCH_FAILED",
|
||||
"获取邮件失败,请检查账号配置",
|
||||
type(exc).__name__,
|
||||
500,
|
||||
str(exc),
|
||||
),
|
||||
}
|
||||
finally:
|
||||
if connection:
|
||||
try:
|
||||
connection.logout()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def fetch_and_detail_imap_with_server(
|
||||
account: str,
|
||||
client_id: str,
|
||||
refresh_token: str,
|
||||
folder: str = "inbox",
|
||||
skip: int = 0,
|
||||
top: int = 1,
|
||||
server: str = IMAP_SERVER_NEW,
|
||||
) -> Dict[str, Any]:
|
||||
"""一次 IMAP 连接完成邮件列表 + 最新一封详情。"""
|
||||
token_result = get_access_token_imap_result(client_id, refresh_token)
|
||||
if not token_result.get("success"):
|
||||
return {
|
||||
"success": False,
|
||||
"error": token_result.get("error"),
|
||||
"emails": [],
|
||||
"detail": None,
|
||||
}
|
||||
|
||||
access_token = token_result["access_token"]
|
||||
connection = None
|
||||
|
||||
try:
|
||||
connection = imaplib.IMAP4_SSL(server, IMAP_PORT)
|
||||
auth_string = f"user={account}\x01auth=Bearer {access_token}\x01\x01".encode("utf-8")
|
||||
connection.authenticate("XOAUTH2", lambda x: auth_string)
|
||||
|
||||
selected = _select_folder(connection, folder)
|
||||
if not selected:
|
||||
return {
|
||||
"success": False,
|
||||
"error": build_error_payload("FOLDER_NOT_FOUND", "文件夹选择失败", "IMAPError", 500, ""),
|
||||
"emails": [],
|
||||
"detail": None,
|
||||
}
|
||||
|
||||
status, messages = connection.search(None, "ALL")
|
||||
if status != "OK" or not messages or not messages[0]:
|
||||
return {"success": True, "emails": [], "detail": None}
|
||||
|
||||
message_ids = messages[0].split()
|
||||
total = len(message_ids)
|
||||
start_idx = max(0, total - skip - top)
|
||||
end_idx = total - skip
|
||||
if start_idx >= end_idx:
|
||||
return {"success": True, "emails": [], "detail": None}
|
||||
|
||||
paged_ids = message_ids[start_idx:end_idx][::-1]
|
||||
emails_data: List[Dict[str, Any]] = []
|
||||
detail = None
|
||||
|
||||
ids_str = b",".join(paged_ids)
|
||||
status, all_data = connection.fetch(ids_str, "(RFC822)")
|
||||
if status != "OK":
|
||||
return {"success": True, "emails": [], "detail": None}
|
||||
|
||||
raw_by_id = {msg_id_str: raw_email for msg_id_str, raw_email in _parse_batch_fetch_response(all_data or [])}
|
||||
|
||||
for i, msg_id in enumerate(paged_ids):
|
||||
msg_id_str = msg_id.decode("ascii", errors="ignore").strip()
|
||||
raw_email = raw_by_id.get(msg_id_str)
|
||||
if raw_email is None:
|
||||
continue
|
||||
|
||||
msg = email.message_from_bytes(raw_email)
|
||||
body_preview = get_email_body(msg)
|
||||
email_item = {
|
||||
"id": msg_id_str,
|
||||
"subject": decode_header_value(msg.get("Subject", "无主题")),
|
||||
"from": decode_header_value(msg.get("From", "未知发件人")),
|
||||
"date": msg.get("Date", "未知时间"),
|
||||
"body_preview": body_preview[:200] + "..." if len(body_preview) > 200 else body_preview,
|
||||
}
|
||||
emails_data.append(email_item)
|
||||
|
||||
if i == 0:
|
||||
raw_text = raw_email.decode("utf-8", errors="replace") if isinstance(raw_email, (bytes, bytearray)) else ""
|
||||
detail = {
|
||||
"id": email_item["id"],
|
||||
"subject": email_item["subject"],
|
||||
"from": email_item["from"],
|
||||
"to": decode_header_value(msg.get("To", "")),
|
||||
"cc": decode_header_value(msg.get("Cc", "")),
|
||||
"date": email_item["date"],
|
||||
"body": get_email_body(msg),
|
||||
"body_html": _get_html_body(msg),
|
||||
"raw_content": raw_text,
|
||||
}
|
||||
|
||||
return {"success": True, "emails": emails_data, "detail": detail}
|
||||
except imaplib.IMAP4.error as exc:
|
||||
return {
|
||||
"success": False,
|
||||
"error": build_error_payload("AUTH_FAILED", "IMAP认证失败", "IMAP4Error", 401, str(exc)),
|
||||
"emails": [],
|
||||
"detail": None,
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"success": False,
|
||||
"error": build_error_payload(
|
||||
"EMAIL_FETCH_FAILED",
|
||||
"获取邮件失败",
|
||||
type(exc).__name__,
|
||||
500,
|
||||
str(exc),
|
||||
),
|
||||
"emails": [],
|
||||
"detail": None,
|
||||
}
|
||||
finally:
|
||||
if connection:
|
||||
try:
|
||||
connection.logout()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def get_emails_imap_concurrent(
|
||||
account: str,
|
||||
client_id: str,
|
||||
refresh_token: str,
|
||||
folder: str = "inbox",
|
||||
skip: int = 0,
|
||||
top: int = 20,
|
||||
servers: tuple = (IMAP_SERVER_NEW, "outlook.office365.com"),
|
||||
) -> Dict[str, Any]:
|
||||
"""并发连接多台 IMAP 服务器,返回第一个成功结果。"""
|
||||
if len(servers) <= 1:
|
||||
return get_emails_imap_with_server(
|
||||
account,
|
||||
client_id,
|
||||
refresh_token,
|
||||
folder,
|
||||
skip,
|
||||
top,
|
||||
servers[0] if servers else IMAP_SERVER_NEW,
|
||||
)
|
||||
|
||||
last_error = None
|
||||
with ThreadPoolExecutor(max_workers=len(servers)) as executor:
|
||||
futures = {
|
||||
executor.submit(
|
||||
get_emails_imap_with_server,
|
||||
account,
|
||||
client_id,
|
||||
refresh_token,
|
||||
folder,
|
||||
skip,
|
||||
top,
|
||||
server,
|
||||
): server
|
||||
for server in servers
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
if result.get("success"):
|
||||
return result
|
||||
last_error = result
|
||||
|
||||
return last_error or {
|
||||
"success": False,
|
||||
"error": {"code": "ALL_SERVERS_FAILED", "message": "所有服务器连接失败"},
|
||||
}
|
||||
|
||||
|
||||
def get_email_detail_imap(
|
||||
account: str,
|
||||
client_id: str,
|
||||
refresh_token: str,
|
||||
message_id: str,
|
||||
folder: str = "inbox",
|
||||
) -> Optional[Dict]:
|
||||
"""使用 IMAP 获取邮件详情(默认使用新版服务器)。"""
|
||||
return get_email_detail_imap_with_server(account, client_id, refresh_token, message_id, folder, IMAP_SERVER_NEW)
|
||||
|
||||
|
||||
def get_email_detail_imap_with_server(
|
||||
account: str,
|
||||
client_id: str,
|
||||
refresh_token: str,
|
||||
message_id: str,
|
||||
folder: str = "inbox",
|
||||
server: str = IMAP_SERVER_NEW,
|
||||
) -> Optional[Dict]:
|
||||
"""使用 IMAP 获取邮件详情(支持指定服务器)。"""
|
||||
access_token = get_access_token_imap(client_id, refresh_token)
|
||||
if not access_token:
|
||||
return None
|
||||
|
||||
connection = None
|
||||
try:
|
||||
connection = imaplib.IMAP4_SSL(server, IMAP_PORT)
|
||||
auth_string = f"user={account}\1auth=Bearer {access_token}\1\1".encode("utf-8")
|
||||
connection.authenticate("XOAUTH2", lambda x: auth_string)
|
||||
|
||||
folder_map = {
|
||||
"inbox": ['"INBOX"', "INBOX"],
|
||||
"junkemail": ['"Junk"', '"Junk Email"', "Junk", '"垃圾邮件"'],
|
||||
"deleteditems": [
|
||||
'"Deleted"',
|
||||
'"Deleted Items"',
|
||||
'"Trash"',
|
||||
"Deleted",
|
||||
'"已删除邮件"',
|
||||
],
|
||||
"trash": [
|
||||
'"Deleted"',
|
||||
'"Deleted Items"',
|
||||
'"Trash"',
|
||||
"Deleted",
|
||||
'"已删除邮件"',
|
||||
],
|
||||
}
|
||||
possible_folders = folder_map.get((folder or "").lower(), ['"INBOX"'])
|
||||
|
||||
selected_folder = None
|
||||
for imap_folder in possible_folders:
|
||||
try:
|
||||
status, response = connection.select(imap_folder, readonly=True)
|
||||
if status == "OK":
|
||||
selected_folder = imap_folder
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not selected_folder:
|
||||
return None
|
||||
|
||||
fetch_id = message_id.encode() if isinstance(message_id, str) else message_id
|
||||
status, msg_data = connection.fetch(fetch_id, "(RFC822)")
|
||||
if status != "OK" or not msg_data or not msg_data[0]:
|
||||
return None
|
||||
|
||||
raw_email = msg_data[0][1]
|
||||
msg = email.message_from_bytes(raw_email)
|
||||
|
||||
raw_text = ""
|
||||
try:
|
||||
raw_text = raw_email.decode("utf-8", errors="replace") if isinstance(raw_email, (bytes, bytearray)) else ""
|
||||
except Exception:
|
||||
raw_text = ""
|
||||
|
||||
body, body_type = get_email_body_and_type(msg)
|
||||
return {
|
||||
"id": message_id,
|
||||
"subject": decode_header_value(msg.get("Subject", "无主题")),
|
||||
"from": decode_header_value(msg.get("From", "未知发件人")),
|
||||
"to": decode_header_value(msg.get("To", "")),
|
||||
"cc": decode_header_value(msg.get("Cc", "")),
|
||||
"date": msg.get("Date", "未知时间"),
|
||||
"body": body,
|
||||
"body_type": body_type,
|
||||
"raw_content": raw_text,
|
||||
}
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
if connection:
|
||||
try:
|
||||
connection.logout()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def delete_emails_imap(
|
||||
email_addr: str,
|
||||
client_id: str,
|
||||
refresh_token: str,
|
||||
message_ids: List[str],
|
||||
server: str,
|
||||
) -> Dict[str, Any]:
|
||||
"""通过 IMAP 删除邮件(永久删除)"""
|
||||
access_token = get_access_token_graph(client_id, refresh_token)
|
||||
if not access_token:
|
||||
return {"success": False, "error": "获取 Access Token 失败"}
|
||||
|
||||
try:
|
||||
auth_string = "user=%s\x01auth=Bearer %s\x01\x01" % (email_addr, access_token)
|
||||
|
||||
imap = imaplib.IMAP4_SSL(server, IMAP_PORT)
|
||||
imap.authenticate("XOAUTH2", lambda x: auth_string.encode("utf-8"))
|
||||
|
||||
imap.select("INBOX")
|
||||
|
||||
# Graph message id 与 IMAP UID 不兼容:保留原行为(暂不支持)
|
||||
return {"success": False, "error": "IMAP 删除暂不支持 (ID 格式不兼容)"}
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
@@ -0,0 +1,598 @@
|
||||
"""
|
||||
统一验证码提取模块(ZER-90)
|
||||
|
||||
将验证码候选生成、评分、门控收敛为单一服务,供 Web API、External API、
|
||||
简洁模式摘要及旧版 verification_extractor 兼容层共用。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from html.parser import HTMLParser
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
# 验证码关键词列表(支持中英文)
|
||||
VERIFICATION_KEYWORDS = [
|
||||
"验证码",
|
||||
"code",
|
||||
"验证",
|
||||
"verification",
|
||||
"OTP",
|
||||
"动态码",
|
||||
"校验码",
|
||||
"verify code",
|
||||
"confirmation code",
|
||||
"security code",
|
||||
"验证码是",
|
||||
"your code",
|
||||
"code is",
|
||||
"激活码",
|
||||
"短信验证码",
|
||||
]
|
||||
|
||||
VERIFICATION_PATTERN = r"\b[A-Z0-9]{4,8}\b"
|
||||
|
||||
# 带连字符字母数字验证码,例如 x.ai 的 84A-KMN
|
||||
HYPHENATED_VERIFICATION_PATTERN = r"(?<![A-Z0-9])([A-Z0-9]{2,4}-[A-Z0-9]{2,4})(?=$|[^A-Z0-9-]|[A-Z][a-z])"
|
||||
|
||||
CODE_CONTEXT_PHRASES = [
|
||||
"validate your email",
|
||||
"validate your email address",
|
||||
"code below",
|
||||
"xai account",
|
||||
"x.ai",
|
||||
"support@x.ai",
|
||||
"verification code",
|
||||
"confirm your email",
|
||||
"verify your email",
|
||||
"your code",
|
||||
"the code below",
|
||||
]
|
||||
|
||||
LINK_PATTERN = r'https?://[^\s<>"{}|\\^`\[\]]+'
|
||||
|
||||
DEFAULT_LINK_KEYWORDS = [
|
||||
"verify",
|
||||
"confirmation",
|
||||
"confirm",
|
||||
"activate",
|
||||
"validation",
|
||||
]
|
||||
|
||||
LINK_CONTEXT_PHRASES = [
|
||||
"verify your email",
|
||||
"verify your account",
|
||||
"verify your address",
|
||||
"confirm your email",
|
||||
"confirm your account",
|
||||
"confirm your address",
|
||||
"activate your email",
|
||||
"activate your account",
|
||||
"email verification",
|
||||
"account verification",
|
||||
"验证您的邮箱",
|
||||
"验证你的邮箱",
|
||||
"验证您的账户",
|
||||
"验证你的账户",
|
||||
"验证您的账号",
|
||||
"验证你的账号",
|
||||
"确认您的邮箱",
|
||||
"确认你的邮箱",
|
||||
"确认您的账户",
|
||||
"确认你的账户",
|
||||
"激活您的账户",
|
||||
"激活你的账户",
|
||||
"激活您的邮箱",
|
||||
"激活你的邮箱",
|
||||
"邮箱验证",
|
||||
"账号验证",
|
||||
"账户验证",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class VerificationPolicy:
|
||||
"""验证码提取策略。"""
|
||||
|
||||
code_regex: str | None = None
|
||||
code_length: str | None = None
|
||||
code_source: str = "all"
|
||||
prefer_link_keywords: List[str] = field(default_factory=lambda: list(DEFAULT_LINK_KEYWORDS))
|
||||
enforce_mutual_exclusion: bool = True
|
||||
apply_confidence_gate: bool = False
|
||||
expected_field: str | None = None # code | link | any
|
||||
|
||||
|
||||
@dataclass
|
||||
class VerificationInput:
|
||||
"""统一邮件输入:subject / 正文 / HTML 分离,避免直接扫描原始 HTML 样式。"""
|
||||
|
||||
subject: str = ""
|
||||
body: str = ""
|
||||
body_preview: str = ""
|
||||
body_html: str = ""
|
||||
html_content: str = ""
|
||||
body_content: str = ""
|
||||
body_content_type: str = ""
|
||||
|
||||
@classmethod
|
||||
def from_email_dict(cls, email: Dict[str, Any]) -> VerificationInput:
|
||||
payload = email or {}
|
||||
return cls(
|
||||
subject=str(payload.get("subject") or "").strip(),
|
||||
body=str(payload.get("body") or "").strip(),
|
||||
body_preview=str(payload.get("body_preview") or "").strip(),
|
||||
body_html=str(payload.get("body_html") or payload.get("html_content") or "").strip(),
|
||||
html_content=str(payload.get("html_content") or "").strip(),
|
||||
body_content=str(payload.get("bodyContent") or "").strip(),
|
||||
body_content_type=str(payload.get("bodyContentType") or "").strip(),
|
||||
)
|
||||
|
||||
def as_legacy_email_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"subject": self.subject,
|
||||
"body": self.body,
|
||||
"body_preview": self.body_preview,
|
||||
"body_html": self.body_html or self.html_content,
|
||||
"html_content": self.html_content,
|
||||
"bodyContent": self.body_content,
|
||||
"bodyContentType": self.body_content_type,
|
||||
}
|
||||
|
||||
|
||||
class HTMLTextExtractor(HTMLParser):
|
||||
"""HTML 转纯文本提取器(跳过 style/script 等不可见节点)。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.text_parts: List[str] = []
|
||||
self._skip_tags = {"style", "script", "head", "meta", "link"}
|
||||
self._current_skip = False
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: Any) -> None:
|
||||
if tag.lower() in self._skip_tags:
|
||||
self._current_skip = True
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag.lower() in self._skip_tags:
|
||||
self._current_skip = False
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if not self._current_skip and data.strip():
|
||||
self.text_parts.append(data.strip())
|
||||
|
||||
def get_text(self) -> str:
|
||||
return " ".join(self.text_parts)
|
||||
|
||||
|
||||
def html_to_visible_text(html_content: str) -> str:
|
||||
if not html_content:
|
||||
return ""
|
||||
parser = HTMLTextExtractor()
|
||||
try:
|
||||
parser.feed(html_content)
|
||||
return html.unescape(parser.get_text() or "").strip()
|
||||
except Exception:
|
||||
return html_content.strip()
|
||||
|
||||
|
||||
def extract_content_text_without_subject(email_input: VerificationInput) -> str:
|
||||
if email_input.body:
|
||||
return email_input.body
|
||||
|
||||
html_raw = email_input.body_html or email_input.html_content
|
||||
if html_raw:
|
||||
return html_to_visible_text(html_raw)
|
||||
|
||||
if email_input.body_content:
|
||||
if email_input.body_content_type.lower() == "html":
|
||||
return html_to_visible_text(email_input.body_content)
|
||||
return email_input.body_content
|
||||
|
||||
if email_input.body_preview:
|
||||
return email_input.body_preview
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def extract_email_text(email: Dict[str, Any]) -> str:
|
||||
email_input = VerificationInput.from_email_dict(email)
|
||||
content = extract_content_text_without_subject(email_input)
|
||||
if content:
|
||||
return content
|
||||
if email_input.subject:
|
||||
return email_input.subject
|
||||
return ""
|
||||
|
||||
|
||||
def _parse_code_length(code_length: str) -> tuple[int, int]:
|
||||
m = re.match(r"^(\d+)-(\d+)$", str(code_length or "").strip())
|
||||
if not m:
|
||||
raise ValueError("code_length 参数无效")
|
||||
min_len = int(m.group(1))
|
||||
max_len = int(m.group(2))
|
||||
if min_len <= 0 or max_len <= 0 or min_len > max_len:
|
||||
raise ValueError("code_length 参数无效")
|
||||
return min_len, max_len
|
||||
|
||||
|
||||
def build_code_regex(*, code_regex: str | None, code_length: str | None) -> re.Pattern[str]:
|
||||
if code_regex:
|
||||
try:
|
||||
return re.compile(code_regex)
|
||||
except re.error as exc:
|
||||
raise ValueError("code_regex 参数无效") from exc
|
||||
|
||||
if code_length:
|
||||
min_len, max_len = _parse_code_length(code_length)
|
||||
return re.compile(rf"\b[A-Za-z0-9]{{{min_len},{max_len}}}\b")
|
||||
|
||||
return re.compile(r"\b\d{4,8}\b")
|
||||
|
||||
|
||||
def _is_valid_hyphenated_code(code: str) -> bool:
|
||||
if not code or "-" not in code:
|
||||
return False
|
||||
|
||||
parts = code.split("-")
|
||||
if len(parts) != 2:
|
||||
return False
|
||||
if not all(part.isalnum() for part in parts):
|
||||
return False
|
||||
|
||||
alnum = "".join(parts)
|
||||
if not (4 <= len(alnum) <= 10):
|
||||
return False
|
||||
if any(c.isdigit() for c in alnum):
|
||||
return True
|
||||
return len(alnum) >= 6 and all(len(part) >= 3 for part in parts) and alnum.isalpha()
|
||||
|
||||
|
||||
def _has_code_context(email_content: str) -> bool:
|
||||
content_lower = email_content.lower()
|
||||
return any(phrase.lower() in content_lower for phrase in CODE_CONTEXT_PHRASES)
|
||||
|
||||
|
||||
def _find_hyphenated_code_in_text(text: str) -> Optional[str]:
|
||||
if not text:
|
||||
return None
|
||||
|
||||
for match in re.finditer(HYPHENATED_VERIFICATION_PATTERN, text, re.IGNORECASE):
|
||||
code = match.group(1)
|
||||
if _is_valid_hyphenated_code(code):
|
||||
return code
|
||||
return None
|
||||
|
||||
|
||||
def smart_extract_hyphenated_verification_code(email_content: str) -> Optional[str]:
|
||||
if not email_content:
|
||||
return None
|
||||
|
||||
content_lower = email_content.lower()
|
||||
for keyword in VERIFICATION_KEYWORDS:
|
||||
keyword_lower = keyword.lower()
|
||||
pos = content_lower.find(keyword_lower)
|
||||
if pos == -1:
|
||||
continue
|
||||
|
||||
start = max(0, pos - 50)
|
||||
end = min(len(email_content), pos + len(keyword) + 50)
|
||||
code = _find_hyphenated_code_in_text(email_content[start:end])
|
||||
if code:
|
||||
return code
|
||||
return None
|
||||
|
||||
|
||||
def fallback_extract_hyphenated_verification_code(email_content: str) -> Optional[str]:
|
||||
if not email_content or not _has_code_context(email_content):
|
||||
return None
|
||||
return _find_hyphenated_code_in_text(email_content)
|
||||
|
||||
|
||||
def smart_extract_verification_code(email_content: str) -> Optional[str]:
|
||||
if not email_content:
|
||||
return None
|
||||
|
||||
content_lower = email_content.lower()
|
||||
for keyword in VERIFICATION_KEYWORDS:
|
||||
keyword_lower = keyword.lower()
|
||||
pos = content_lower.find(keyword_lower)
|
||||
if pos == -1:
|
||||
continue
|
||||
|
||||
start = max(0, pos - 50)
|
||||
end = min(len(email_content), pos + len(keyword) + 50)
|
||||
context = email_content[start:end]
|
||||
matches = re.findall(VERIFICATION_PATTERN, context, re.IGNORECASE)
|
||||
for match in matches:
|
||||
if any(c.isdigit() for c in match):
|
||||
return match
|
||||
|
||||
return smart_extract_hyphenated_verification_code(email_content)
|
||||
|
||||
|
||||
def fallback_extract_verification_code(email_content: str) -> Optional[str]:
|
||||
if not email_content:
|
||||
return None
|
||||
|
||||
filtered: List[str] = []
|
||||
for match in re.findall(VERIFICATION_PATTERN, email_content, re.IGNORECASE):
|
||||
if not any(c.isdigit() for c in match):
|
||||
continue
|
||||
|
||||
if match.isdigit() and len(match) == 4:
|
||||
year = int(match)
|
||||
if 1900 <= year <= 2100:
|
||||
continue
|
||||
hour = int(match[:2])
|
||||
minute = int(match[2:])
|
||||
if 0 <= hour <= 23 and 0 <= minute <= 59:
|
||||
continue
|
||||
if 2020 <= year <= 2030:
|
||||
continue
|
||||
|
||||
filtered.append(match)
|
||||
|
||||
if filtered:
|
||||
return filtered[0]
|
||||
|
||||
return fallback_extract_hyphenated_verification_code(email_content)
|
||||
|
||||
|
||||
def smart_extract_code_by_keywords(email_content: str, code_re: re.Pattern[str]) -> Optional[str]:
|
||||
if not email_content:
|
||||
return None
|
||||
|
||||
content_lower = email_content.lower()
|
||||
for keyword in VERIFICATION_KEYWORDS:
|
||||
keyword_lower = keyword.lower()
|
||||
pos = content_lower.find(keyword_lower)
|
||||
if pos == -1:
|
||||
continue
|
||||
|
||||
start = max(0, pos - 50)
|
||||
end = min(len(email_content), pos + len(keyword) + 50)
|
||||
context = email_content[start:end]
|
||||
for match in code_re.finditer(context):
|
||||
value = match.group(0)
|
||||
if value and any(c.isdigit() for c in value):
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def fallback_extract_code(email_content: str, code_re: re.Pattern[str]) -> Optional[str]:
|
||||
if not email_content:
|
||||
return None
|
||||
|
||||
candidates: List[str] = []
|
||||
for match in code_re.finditer(email_content):
|
||||
value = match.group(0) or ""
|
||||
if not value or not any(c.isdigit() for c in value):
|
||||
continue
|
||||
|
||||
if value.isdigit() and len(value) == 4:
|
||||
year = int(value)
|
||||
if 1900 <= year <= 2100:
|
||||
continue
|
||||
hour = int(value[:2])
|
||||
minute = int(value[2:])
|
||||
if 0 <= hour <= 23 and 0 <= minute <= 59:
|
||||
continue
|
||||
if 2020 <= year <= 2030:
|
||||
continue
|
||||
|
||||
candidates.append(value)
|
||||
|
||||
return candidates[0] if candidates else None
|
||||
|
||||
|
||||
def extract_links(email_content: str) -> List[str]:
|
||||
if not email_content:
|
||||
return []
|
||||
|
||||
cleaned_links = [link.rstrip(".,;:!?)>'\"") for link in re.findall(LINK_PATTERN, email_content, re.IGNORECASE)]
|
||||
seen: set[str] = set()
|
||||
unique_links: List[str] = []
|
||||
for link in cleaned_links:
|
||||
if link not in seen:
|
||||
seen.add(link)
|
||||
unique_links.append(link)
|
||||
return unique_links
|
||||
|
||||
|
||||
def pick_preferred_link(links: List[str], prefer_link_keywords: List[str]) -> Optional[str]:
|
||||
if not links:
|
||||
return None
|
||||
|
||||
keywords = [keyword.lower() for keyword in (prefer_link_keywords or []) if keyword]
|
||||
if keywords:
|
||||
for keyword in keywords:
|
||||
for link in links:
|
||||
if keyword in (link or "").lower():
|
||||
return link
|
||||
return links[0]
|
||||
|
||||
|
||||
def build_source_text(email_input: VerificationInput, *, code_source: str) -> tuple[str, str]:
|
||||
subject = email_input.subject
|
||||
content = extract_content_text_without_subject(email_input)
|
||||
html_raw = email_input.body_html or email_input.html_content
|
||||
|
||||
source = str(code_source or "all").strip().lower()
|
||||
if source == "subject":
|
||||
return subject, "subject"
|
||||
if source == "content":
|
||||
return content, "content"
|
||||
if source == "html":
|
||||
return (html_to_visible_text(html_raw) if html_raw else ""), "html"
|
||||
return f"{subject} {content}".strip(), "all"
|
||||
|
||||
|
||||
def extract_verification_code_from_text(
|
||||
source_text: str,
|
||||
*,
|
||||
code_regex: str | None,
|
||||
code_length: str | None,
|
||||
) -> tuple[Optional[str], str]:
|
||||
code_re = build_code_regex(code_regex=code_regex, code_length=code_length)
|
||||
caller_directed_code = bool(code_regex)
|
||||
|
||||
verification_code = smart_extract_code_by_keywords(source_text, code_re)
|
||||
code_confidence = "high" if verification_code else "low"
|
||||
|
||||
if not verification_code:
|
||||
verification_code = fallback_extract_code(source_text, code_re)
|
||||
if verification_code and caller_directed_code:
|
||||
code_confidence = "high"
|
||||
|
||||
if not verification_code:
|
||||
verification_code = smart_extract_hyphenated_verification_code(source_text)
|
||||
if verification_code:
|
||||
code_confidence = "high"
|
||||
|
||||
if not verification_code:
|
||||
verification_code = fallback_extract_hyphenated_verification_code(source_text)
|
||||
if verification_code:
|
||||
code_confidence = "high"
|
||||
|
||||
return verification_code, code_confidence
|
||||
|
||||
|
||||
def extract_verification(
|
||||
email_input: VerificationInput,
|
||||
policy: VerificationPolicy | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
统一验证码/链接提取入口。
|
||||
|
||||
返回字段与 extract_verification_info_with_options 兼容。
|
||||
"""
|
||||
active_policy = policy or VerificationPolicy()
|
||||
source_text, match_source = build_source_text(email_input, code_source=active_policy.code_source)
|
||||
subject = email_input.subject
|
||||
content = extract_content_text_without_subject(email_input)
|
||||
html_raw = email_input.body_html or email_input.html_content
|
||||
|
||||
verification_code, code_confidence = extract_verification_code_from_text(
|
||||
source_text,
|
||||
code_regex=active_policy.code_regex,
|
||||
code_length=active_policy.code_length,
|
||||
)
|
||||
|
||||
links = extract_links(f"{subject} {content} {html_raw}".strip())
|
||||
prefer_keywords = active_policy.prefer_link_keywords or DEFAULT_LINK_KEYWORDS
|
||||
|
||||
verification_link = None
|
||||
link_confidence = "low"
|
||||
should_pick_link = (not active_policy.enforce_mutual_exclusion) or (not verification_code)
|
||||
if should_pick_link:
|
||||
verification_link = pick_preferred_link(links, prefer_keywords)
|
||||
if verification_link:
|
||||
for keyword in prefer_keywords:
|
||||
if keyword and keyword.lower() in verification_link.lower():
|
||||
link_confidence = "high"
|
||||
break
|
||||
if link_confidence != "high":
|
||||
full_text_lower = f"{subject} {content}".lower()
|
||||
for phrase in LINK_CONTEXT_PHRASES:
|
||||
if phrase.lower() in full_text_lower:
|
||||
link_confidence = "high"
|
||||
break
|
||||
|
||||
confidence = "high" if code_confidence == "high" or link_confidence == "high" else "low"
|
||||
|
||||
parts: List[str] = []
|
||||
if verification_code:
|
||||
parts.append(verification_code)
|
||||
if verification_link:
|
||||
parts.append(verification_link)
|
||||
formatted = " ".join(parts) if parts else None
|
||||
|
||||
result = {
|
||||
"verification_code": verification_code,
|
||||
"verification_link": verification_link,
|
||||
"links": links,
|
||||
"formatted": formatted,
|
||||
"match_source": match_source,
|
||||
"confidence": confidence,
|
||||
"code_confidence": code_confidence,
|
||||
"link_confidence": link_confidence,
|
||||
}
|
||||
|
||||
if active_policy.apply_confidence_gate:
|
||||
result = apply_confidence_gate(result, enforce_mutual_exclusion=active_policy.enforce_mutual_exclusion)
|
||||
|
||||
expected_field = str(active_policy.expected_field or "").strip().lower()
|
||||
if expected_field == "code":
|
||||
result["verification_link"] = None
|
||||
result["link_confidence"] = "low"
|
||||
result["formatted"] = result.get("verification_code") or None
|
||||
elif expected_field == "link":
|
||||
result["verification_code"] = None
|
||||
result["code_confidence"] = "low"
|
||||
result["formatted"] = result.get("verification_link") or None
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def apply_confidence_gate(extracted: Dict[str, Any], *, enforce_mutual_exclusion: bool = True) -> Dict[str, Any]:
|
||||
result = dict(extracted)
|
||||
|
||||
if result.get("code_confidence") != "high":
|
||||
result["verification_code"] = None
|
||||
if result.get("link_confidence") != "high":
|
||||
result["verification_link"] = None
|
||||
|
||||
if enforce_mutual_exclusion and result.get("verification_code"):
|
||||
result["verification_link"] = None
|
||||
result["link_confidence"] = "low"
|
||||
|
||||
parts = [value for value in (result.get("verification_code"), result.get("verification_link")) if value]
|
||||
result["formatted"] = " ".join(parts) if parts else None
|
||||
result["confidence"] = (
|
||||
"high" if result.get("code_confidence") == "high" or result.get("link_confidence") == "high" else "low"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def policy_from_resolved(
|
||||
resolved: Dict[str, Any] | None,
|
||||
*,
|
||||
code_source: str = "all",
|
||||
enforce_mutual_exclusion: bool = True,
|
||||
apply_confidence_gate: bool = False,
|
||||
expected_field: str | None = None,
|
||||
) -> VerificationPolicy:
|
||||
payload = resolved or {}
|
||||
return VerificationPolicy(
|
||||
code_regex=payload.get("code_regex"),
|
||||
code_length=payload.get("code_length"),
|
||||
code_source=code_source,
|
||||
enforce_mutual_exclusion=enforce_mutual_exclusion,
|
||||
apply_confidence_gate=apply_confidence_gate,
|
||||
expected_field=expected_field,
|
||||
)
|
||||
|
||||
|
||||
def extract_verification_from_email_dict(
|
||||
email: Dict[str, Any],
|
||||
*,
|
||||
code_regex: str | None = None,
|
||||
code_length: str | None = None,
|
||||
code_source: str = "all",
|
||||
prefer_link_keywords: list[str] | None = None,
|
||||
enforce_mutual_exclusion: bool = True,
|
||||
apply_confidence_gate_after: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""兼容旧 extract_verification_info_with_options 签名的薄封装。"""
|
||||
policy = VerificationPolicy(
|
||||
code_regex=code_regex,
|
||||
code_length=code_length,
|
||||
code_source=code_source,
|
||||
prefer_link_keywords=list(prefer_link_keywords or DEFAULT_LINK_KEYWORDS),
|
||||
enforce_mutual_exclusion=enforce_mutual_exclusion,
|
||||
apply_confidence_gate=apply_confidence_gate_after,
|
||||
)
|
||||
return extract_verification(VerificationInput.from_email_dict(email), policy)
|
||||
Reference in New Issue
Block a user