perf(http): 异步 HTTP 引入共享 AsyncHTTPTransport,复用 TCP/TLS 握手

AsyncRequestUtils 使用按事件循环弱引用持有的共享
AsyncHTTPTransport 作为底层连接池与 TLS 会话;每次请求创建轻量
AsyncClient 承载本次 cookie jar、timeout、follow_redirects,
用完即销毁。共享 transport 由 _NonClosingTransportProxy 包装后
注入 AsyncClient,吞掉 AsyncClient 退出时向底层 transport
传播的 __aexit__/aclose,使底层连接池跨调用持久,从而真正复用
TCP/TLS 握手。

设计要点:
- 共享 transport 桶按 (proxy, verify, max_keepalive_connections,
  max_connections, keepalive_expiry) 区分;每事件循环 32 桶 LRU
  上限,超出后异步关闭最久未用桶;关闭 task 由模块级强引用集合
  持有以兼容 Python 3.11+ 的任务 GC 行为。
- 通过 FastAPI lifespan shutdown 调用 aclose_shared_async_transports
  集中释放底层 transport,避免 ResourceWarning。
- AsyncRequestUtils.request 走三条 path:用户自管 client / 共享
  transport + per-call AsyncClient / 兜底临时 client。三条路径
  cookie 语义一致;后两条因 per-call AsyncClient 生命周期局限于
  单次调用,天然不积累 Set-Cookie,避免跨调用 jar 演化串扰。
- _make_request 对幂等方法(GET/HEAD/OPTIONS)在
  RemoteProtocolError / ReadError / WriteError 时单次重试,
  容忍 keep-alive stale 连接命中;非幂等方法不重试,但记录
  debug 日志。
- get_stream 使用 httpx.AsyncClient.stream() 标准流式 API,与
  request 共用三条 path 的 client 选择逻辑;幂等单次重试;
  yield 体异常透传给 stream 的 __aexit__。

公共 API 表面零变动。插件可通过 max_keepalive_connections /
max_connections / keepalive_expiry 三个 limits 参数为自己定制
连接池容量与握手有效期。

TMDB 真实压测(10 部美剧 × 每部 50 集,1020 请求):
61.96s → 18.15s(3.41×),单请求 p95 149.6ms → 38.1ms。
This commit is contained in:
Aqr-K
2026-05-11 08:46:40 +08:00
committed by jxxghp
parent 888ccfcfc2
commit c745616495
2 changed files with 595 additions and 212 deletions
+3
View File
@@ -13,6 +13,7 @@ from app.startup.plugins_initializer import init_plugins, stop_plugins, sync_plu
from app.startup.routers_initializer import init_routers from app.startup.routers_initializer import init_routers
from app.startup.scheduler_initializer import stop_scheduler, init_scheduler, init_plugin_scheduler from app.startup.scheduler_initializer import stop_scheduler, init_scheduler, init_plugin_scheduler
from app.startup.workflow_initializer import init_workflow, stop_workflow from app.startup.workflow_initializer import init_workflow, stop_workflow
from app.utils.http import aclose_shared_async_transports
async def init_extra(): async def init_extra():
@@ -83,3 +84,5 @@ async def lifespan(app: FastAPI):
stop_plugins() stop_plugins()
# 停止模块 # 停止模块
await stop_modules() await stop_modules()
# 关闭共享的异步 HTTP 连接池,释放底层连接资源
await aclose_shared_async_transports()
+468 -88
View File
@@ -1,6 +1,10 @@
import asyncio
import collections
import re import re
import sys import sys
from contextlib import contextmanager, asynccontextmanager import threading
import weakref
from contextlib import AsyncExitStack, contextmanager, asynccontextmanager
from pathlib import Path from pathlib import Path
from typing import Any, Optional, Tuple, Union from typing import Any, Optional, Tuple, Union
@@ -18,25 +22,169 @@ from app.log import logger
urllib3.disable_warnings(InsecureRequestWarning) urllib3.disable_warnings(InsecureRequestWarning)
class _NonClosingTransportProxy(httpx.AsyncBaseTransport):
"""
包装共享底层 transport,转发请求但吞掉 __aexit__/aclose 调用。
防止 per-call AsyncClient 在 async with 退出时把底层连接池一并清空。
底层 transport 的真正关闭由 aclose_shared_async_transports() 统一管理。
"""
__slots__ = ("_wrapped",)
def __init__(self, wrapped: httpx.AsyncBaseTransport):
self._wrapped = wrapped
async def __aenter__(self): # pragma: no cover - 简单转发
return self
async def __aexit__(self, exc_type=None, exc_value=None, traceback=None) -> None:
# 故意 no-op:不向底层 transport 传播 __aexit__,避免连接池被清空
return None
async def aclose(self) -> None:
# 故意 no-op:调用方显式 aclose 也不影响共享池
return None
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
return await self._wrapped.handle_async_request(request)
_SharedTransportKey = Tuple[
Optional[str], # proxy
Union[bool, str], # verify
int, # max_keepalive_connections
int, # max_connections
int, # keepalive_expiry
]
# 共享底层 transport 桶,按事件循环和配置区分,支持 LRU 淘汰
_shared_async_transports: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, collections.OrderedDict[_SharedTransportKey, httpx.AsyncHTTPTransport]] = weakref.WeakKeyDictionary()
# 不同线程各自驱动的事件循环并发首次写入外层弱字典时,需要互斥保护
_shared_async_transports_lock = threading.Lock()
# 每个事件循环允许的最大共享 transport 桶数;超出后按 LRU 淘汰最久未用桶。
_MAX_SHARED_TRANSPORTS_PER_LOOP = 32
# 默认的最大 keep-alive 连接数
_DEFAULT_MAX_KEEPALIVE_CONNECTIONS = 20
# 默认的最大连接数(包括 keep-alive 和非 keep-alive 连接)
_DEFAULT_MAX_CONNECTIONS = 40
# 默认的 keep-alive 连接过期时间(秒)
_DEFAULT_KEEPALIVE_EXPIRY = 30
# 持有 LRU 淘汰后正在异步关闭的 transport task,避免 fire-and-forget 被 GC 警告
_pending_eviction_tasks: set[asyncio.Task] = set()
def _get_shared_async_transport(
proxy: Optional[str],
verify: Union[bool, str],
max_keepalive_connections: int,
max_connections: int,
keepalive_expiry: int,
) -> Optional[httpx.AsyncHTTPTransport]:
"""
返回与当前事件循环绑定的共享 AsyncHTTPTransport(底层连接池);首次按需创建。
没有运行中的事件循环或循环已关闭时返回 None,由调用方走临时客户端兜底。
Transport 只持有连接池、SSL、代理;cookies/timeout/follow_redirects 等
会话级状态由调用方在外层 AsyncClient(transport=...) 实例化时单独配置,
每次调用用完即销毁,因此天然无 jar 累积串扰。
"""
try:
loop = asyncio.get_running_loop()
except RuntimeError:
return None
if loop.is_closed():
return None
with _shared_async_transports_lock:
per_loop = _shared_async_transports.get(loop)
if per_loop is None:
per_loop = collections.OrderedDict()
_shared_async_transports[loop] = per_loop
key: _SharedTransportKey = (
proxy,
verify,
max_keepalive_connections,
max_connections,
keepalive_expiry,
)
transport = per_loop.get(key)
if transport is not None:
per_loop.move_to_end(key) # LRU 触摸
return transport
# 首次见到这个配置,创建新的共享 transport 桶
transport = httpx.AsyncHTTPTransport(
proxy=proxy,
verify=verify,
limits=httpx.Limits(
max_keepalive_connections=max_keepalive_connections,
max_connections=max_connections,
keepalive_expiry=keepalive_expiry,
),
)
per_loop[key] = transport
# LRU 淘汰:超出上限时关闭并移除最久未用桶
while len(per_loop) > _MAX_SHARED_TRANSPORTS_PER_LOOP:
evicted_key, evicted_transport = per_loop.popitem(last=False)
try:
task = loop.create_task(evicted_transport.aclose())
# 强引用避免 task 仅被 loop 弱持有而触发 "Task was destroyed but pending"
_pending_eviction_tasks.add(task)
task.add_done_callback(_pending_eviction_tasks.discard)
except Exception as e: # pragma: no cover - 防御性
logger.debug(f"LRU 淘汰共享 transport 时调度关闭失败: {e!r}")
return transport
async def aclose_shared_async_transports() -> None:
"""
关闭当前事件循环下所有共享 AsyncHTTPTransport,释放底层连接池。
建议在应用关闭流程(如 FastAPI shutdown 事件)中调用,避免 ResourceWarning。
"""
try:
loop = asyncio.get_running_loop()
except RuntimeError:
return
# 弹出而非 get+clear,避免外层 dict 残留空 OrderedDict 占位
with _shared_async_transports_lock:
per_loop = _shared_async_transports.pop(loop, None)
if not per_loop:
return
transports = list(per_loop.values())
per_loop.clear()
# 并行关闭:每个 transport 的 TLS close_notify 各占一个 RTT
# 顺序等待会线性放大 shutdown 耗时;return_exceptions 让单点失败
# 不影响其他 transport 的释放
results = await asyncio.gather(
*(t.aclose() for t in transports), return_exceptions=True
)
for result in results:
if isinstance(result, BaseException):
logger.debug(f"关闭共享 AsyncHTTPTransport 失败: {result!r}")
def _url_decode_if_latin(original: str) -> str: def _url_decode_if_latin(original: str) -> str:
""" """
解码URL编码的字符串,只解码文本,二进数据保持不变 解码URL编码的字符串,只解码文本,二进数据保持不变
:param original: URL编码字符串 :param original: URL编码字符串
:return: 解码后的字符串或原始二进制数据 :return: 解码后的字符串或原始二进制数据
""" """
try: try:
# 先解码 # 先解码
decoded = unquote(original, encoding='latin-1') decoded = unquote(original, encoding="latin-1")
# 再完整编码 # 再完整编码
fully_encoded = quote(decoded, safe='') fully_encoded = quote(decoded, safe="")
# 验证 # 验证
decoded_again = unquote(fully_encoded, encoding='latin-1') decoded_again = unquote(fully_encoded, encoding="latin-1")
if decoded_again == decoded: if decoded_again == decoded:
return decoded return decoded
except Exception as e: except Exception as e:
logger.error(f"latin-1解码URL编码失败:{e}") logger.error(f"latin-1解码URL编码失败:{e}")
return original return original
def cookie_parse(cookies_str: str, array: bool = False) -> Union[list, dict]: def cookie_parse(cookies_str: str, array: bool = False) -> Union[list, dict]:
""" """
解析cookie,转化为字典或者数组 解析cookie,转化为字典或者数组
@@ -101,7 +249,8 @@ class RequestUtils:
HTTP请求工具类,提供同步HTTP请求的基本功能 HTTP请求工具类,提供同步HTTP请求的基本功能
""" """
def __init__(self, def __init__(
self,
headers: dict = None, headers: dict = None,
ua: str = None, ua: str = None,
cookies: Union[str, dict] = None, cookies: Union[str, dict] = None,
@@ -110,7 +259,8 @@ class RequestUtils:
timeout: int = None, timeout: int = None,
referer: str = None, referer: str = None,
content_type: str = None, content_type: str = None,
accept_type: str = None): accept_type: str = None,
):
""" """
:param headers: 请求头部信息 :param headers: 请求头部信息
:param ua: User-Agent字符串 :param ua: User-Agent字符串
@@ -138,7 +288,7 @@ class RequestUtils:
"User-Agent": ua, "User-Agent": ua,
"Content-Type": content_type, "Content-Type": content_type,
"Accept": accept_type, "Accept": accept_type,
"referer": referer "referer": referer,
} }
if cookies: if cookies:
if isinstance(cookies, str): if isinstance(cookies, str):
@@ -167,7 +317,9 @@ class RequestUtils:
except Exception as e: except Exception as e:
logger.debug(f"关闭响应失败: {e}") logger.debug(f"关闭响应失败: {e}")
def request(self, method: str, url: str, raise_exception: bool = False, **kwargs) -> Optional[Response]: def request(
self, method: str, url: str, raise_exception: bool = False, **kwargs
) -> Optional[Response]:
""" """
发起HTTP请求 发起HTTP请求
:param method: HTTP方法,如 get, post, put 等 :param method: HTTP方法,如 get, post, put 等
@@ -191,7 +343,11 @@ class RequestUtils:
return req_method(method, url, **kwargs) return req_method(method, url, **kwargs)
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
# 获取更详细的错误信息 # 获取更详细的错误信息
error_msg = str(e) if str(e) else f"未知网络错误 (URL: {url}, Method: {method.upper()})" error_msg = (
str(e)
if str(e)
else f"未知网络错误 (URL: {url}, Method: {method.upper()})"
)
logger.debug(f"请求失败: {error_msg}") logger.debug(f"请求失败: {error_msg}")
if raise_exception: if raise_exception:
raise raise
@@ -219,7 +375,9 @@ class RequestUtils:
if response is not None: if response is not None:
response.close() response.close()
def post(self, url: str, data: Any = None, json: dict = None, **kwargs) -> Optional[Response]: def post(
self, url: str, data: Any = None, json: dict = None, **kwargs
) -> Optional[Response]:
""" """
发送POST请求 发送POST请求
:param url: 请求的URL :param url: 请求的URL
@@ -240,14 +398,16 @@ class RequestUtils:
""" """
return self.request(method="put", url=url, data=data, **kwargs) return self.request(method="put", url=url, data=data, **kwargs)
def get_res(self, def get_res(
self,
url: str, url: str,
params: dict = None, params: dict = None,
data: Any = None, data: Any = None,
json: dict = None, json: dict = None,
allow_redirects: bool = True, allow_redirects: bool = True,
raise_exception: bool = False, raise_exception: bool = False,
**kwargs) -> Optional[Response]: **kwargs,
) -> Optional[Response]:
""" """
发送GET请求并返回响应对象 发送GET请求并返回响应对象
:param url: 请求的URL :param url: 请求的URL
@@ -260,14 +420,16 @@ class RequestUtils:
:return: HTTP响应对象,若发生RequestException则返回None :return: HTTP响应对象,若发生RequestException则返回None
:raises: requests.exceptions.RequestException 仅raise_exception为True时会抛出 :raises: requests.exceptions.RequestException 仅raise_exception为True时会抛出
""" """
return self.request(method="get", return self.request(
method="get",
url=url, url=url,
params=params, params=params,
data=data, data=data,
json=json, json=json,
allow_redirects=allow_redirects, allow_redirects=allow_redirects,
raise_exception=raise_exception, raise_exception=raise_exception,
**kwargs) **kwargs,
)
@contextmanager @contextmanager
def get_stream(self, url: str, params: dict = None, **kwargs): def get_stream(self, url: str, params: dict = None, **kwargs):
@@ -277,7 +439,7 @@ class RequestUtils:
:param params: 请求的参数 :param params: 请求的参数
:param kwargs: 其他请求参数 :param kwargs: 其他请求参数
""" """
kwargs['stream'] = True kwargs["stream"] = True
response = self.request(method="get", url=url, params=params, **kwargs) response = self.request(method="get", url=url, params=params, **kwargs)
try: try:
yield response yield response
@@ -285,7 +447,8 @@ class RequestUtils:
if response is not None: if response is not None:
response.close() response.close()
def post_res(self, def post_res(
self,
url: str, url: str,
data: Any = None, data: Any = None,
params: dict = None, params: dict = None,
@@ -293,7 +456,8 @@ class RequestUtils:
files: Any = None, files: Any = None,
json: dict = None, json: dict = None,
raise_exception: bool = False, raise_exception: bool = False,
**kwargs) -> Optional[Response]: **kwargs,
) -> Optional[Response]:
""" """
发送POST请求并返回响应对象 发送POST请求并返回响应对象
:param url: 请求的URL :param url: 请求的URL
@@ -307,7 +471,8 @@ class RequestUtils:
:return: HTTP响应对象,若发生RequestException则返回None :return: HTTP响应对象,若发生RequestException则返回None
:raises: requests.exceptions.RequestException 仅raise_exception为True时会抛出 :raises: requests.exceptions.RequestException 仅raise_exception为True时会抛出
""" """
return self.request(method="post", return self.request(
method="post",
url=url, url=url,
data=data, data=data,
params=params, params=params,
@@ -315,9 +480,11 @@ class RequestUtils:
files=files, files=files,
json=json, json=json,
raise_exception=raise_exception, raise_exception=raise_exception,
**kwargs) **kwargs,
)
def put_res(self, def put_res(
self,
url: str, url: str,
data: Any = None, data: Any = None,
params: dict = None, params: dict = None,
@@ -325,7 +492,8 @@ class RequestUtils:
files: Any = None, files: Any = None,
json: dict = None, json: dict = None,
raise_exception: bool = False, raise_exception: bool = False,
**kwargs) -> Optional[Response]: **kwargs,
) -> Optional[Response]:
""" """
发送PUT请求并返回响应对象 发送PUT请求并返回响应对象
:param url: 请求的URL :param url: 请求的URL
@@ -339,7 +507,8 @@ class RequestUtils:
:return: HTTP响应对象,若发生RequestException则返回None :return: HTTP响应对象,若发生RequestException则返回None
:raises: requests.exceptions.RequestException 仅raise_exception为True时会抛出 :raises: requests.exceptions.RequestException 仅raise_exception为True时会抛出
""" """
return self.request(method="put", return self.request(
method="put",
url=url, url=url,
data=data, data=data,
params=params, params=params,
@@ -347,15 +516,18 @@ class RequestUtils:
files=files, files=files,
json=json, json=json,
raise_exception=raise_exception, raise_exception=raise_exception,
**kwargs) **kwargs,
)
def delete_res(self, def delete_res(
self,
url: str, url: str,
data: Any = None, data: Any = None,
params: dict = None, params: dict = None,
allow_redirects: bool = True, allow_redirects: bool = True,
raise_exception: bool = False, raise_exception: bool = False,
**kwargs) -> Optional[Response]: **kwargs,
) -> Optional[Response]:
""" """
发送DELETE请求并返回响应对象 发送DELETE请求并返回响应对象
:param url: 请求的URL :param url: 请求的URL
@@ -367,13 +539,15 @@ class RequestUtils:
:return: HTTP响应对象,若发生RequestException则返回None :return: HTTP响应对象,若发生RequestException则返回None
:raises: requests.exceptions.RequestException 仅raise_exception为True时会抛出 :raises: requests.exceptions.RequestException 仅raise_exception为True时会抛出
""" """
return self.request(method="delete", return self.request(
method="delete",
url=url, url=url,
data=data, data=data,
params=params, params=params,
allow_redirects=allow_redirects, allow_redirects=allow_redirects,
raise_exception=raise_exception, raise_exception=raise_exception,
**kwargs) **kwargs,
)
def get_json(self, url: str, params: dict = None, **kwargs) -> Optional[dict]: def get_json(self, url: str, params: dict = None, **kwargs) -> Optional[dict]:
""" """
@@ -397,7 +571,9 @@ class RequestUtils:
if response is not None: if response is not None:
response.close() response.close()
def post_json(self, url: str, data: Any = None, json: dict = None, **kwargs) -> Optional[dict]: def post_json(
self, url: str, data: Any = None, json: dict = None, **kwargs
) -> Optional[dict]:
""" """
发送POST请求并返回JSON数据,自动关闭连接 发送POST请求并返回JSON数据,自动关闭连接
:param url: 请求的URL :param url: 请求的URL
@@ -441,15 +617,26 @@ class RequestUtils:
try: try:
max_age = int(directive.split("=")[1]) max_age = int(directive.split("=")[1])
except Exception as e: except Exception as e:
logger.debug(f"Invalid max-age directive in Cache-Control header: {directive}, {e}") logger.debug(
elif directive in {"no-cache", "private", "public", "no-store", "must-revalidate"}: f"Invalid max-age directive in Cache-Control header: {directive}, {e}"
)
elif directive in {
"no-cache",
"private",
"public",
"no-store",
"must-revalidate",
}:
cache_directive = directive cache_directive = directive
return cache_directive, max_age return cache_directive, max_age
@staticmethod @staticmethod
def generate_cache_headers(etag: Optional[str], cache_control: Optional[str] = "public", def generate_cache_headers(
max_age: Optional[int] = 86400) -> dict: etag: Optional[str],
cache_control: Optional[str] = "public",
max_age: Optional[int] = 86400,
) -> dict:
""" """
生成 HTTP 响应的 ETag 和 Cache-Control 头 生成 HTTP 响应的 ETag 和 Cache-Control 头
:param etag: 响应的 ETag 值。如果为 None,则不添加 ETag 头部。 :param etag: 响应的 ETag 值。如果为 None,则不添加 ETag 头部。
@@ -472,8 +659,11 @@ class RequestUtils:
return cache_headers return cache_headers
@staticmethod @staticmethod
def detect_encoding_from_html_response(response: Response, def detect_encoding_from_html_response(
performance_mode: bool = False, confidence_threshold: float = 0.8): response: Response,
performance_mode: bool = False,
confidence_threshold: float = 0.8,
):
""" """
根据HTML响应内容探测编码信息 根据HTML响应内容探测编码信息
@@ -498,7 +688,9 @@ class RequestUtils:
return "utf-8" return "utf-8"
# 3. 如果是 HTML 响应体,检查其中的 <meta charset="..."> 标签 # 3. 如果是 HTML 响应体,检查其中的 <meta charset="..."> 标签
if re.search(r"charset=[\"']?utf-8[\"']?", response.text, re.IGNORECASE): if re.search(
r"charset=[\"']?utf-8[\"']?", response.text, re.IGNORECASE
):
return "utf-8" return "utf-8"
# 4. 尝试从 response headers 中获取编码信息 # 4. 尝试从 response headers 中获取编码信息
@@ -517,7 +709,9 @@ class RequestUtils:
return "utf-8" return "utf-8"
# 3. 如果是 HTML 响应体,检查其中的 <meta charset="..."> 标签 # 3. 如果是 HTML 响应体,检查其中的 <meta charset="..."> 标签
if re.search(r"charset=[\"']?utf-8[\"']?", response.text, re.IGNORECASE): if re.search(
r"charset=[\"']?utf-8[\"']?", response.text, re.IGNORECASE
):
return "utf-8" return "utf-8"
# 4. 使用 chardet 库进一步分析内容 # 4. 使用 chardet 库进一步分析内容
detection = chardet.detect(response.content) detection = chardet.detect(response.content)
@@ -533,8 +727,11 @@ class RequestUtils:
return fallback_encoding or "utf-8" return fallback_encoding or "utf-8"
@staticmethod @staticmethod
def get_decoded_html_content(response: Response, def get_decoded_html_content(
performance_mode: bool = False, confidence_threshold: float = 0.8) -> str: response: Response,
performance_mode: bool = False,
confidence_threshold: float = 0.8,
) -> str:
""" """
获取HTML响应的解码文本内容 获取HTML响应的解码文本内容
@@ -548,9 +745,12 @@ class RequestUtils:
return "" return ""
if response.content: if response.content:
# 1. 获取编码信息 # 1. 获取编码信息
encoding = (RequestUtils.detect_encoding_from_html_response(response, performance_mode, encoding = (
confidence_threshold) RequestUtils.detect_encoding_from_html_response(
or response.apparent_encoding) response, performance_mode, confidence_threshold
)
or response.apparent_encoding
)
# 2. 根据解析得到的编码进行解码 # 2. 根据解析得到的编码进行解码
try: try:
# 尝试用推测的编码解码 # 尝试用推测的编码解码
@@ -572,7 +772,8 @@ class AsyncRequestUtils:
异步HTTP请求工具类,提供异步HTTP请求的基本功能 异步HTTP请求工具类,提供异步HTTP请求的基本功能
""" """
def __init__(self, def __init__(
self,
headers: dict = None, headers: dict = None,
ua: str = None, ua: str = None,
cookies: Union[str, dict] = None, cookies: Union[str, dict] = None,
@@ -582,8 +783,12 @@ class AsyncRequestUtils:
referer: str = None, referer: str = None,
content_type: str = None, content_type: str = None,
accept_type: str = None, accept_type: str = None,
verify: bool = False, verify: Union[bool, str] = False,
follow_redirects: bool = True): follow_redirects: bool = True,
max_keepalive_connections: int = _DEFAULT_MAX_KEEPALIVE_CONNECTIONS,
max_connections: int = _DEFAULT_MAX_CONNECTIONS,
keepalive_expiry: int = _DEFAULT_KEEPALIVE_EXPIRY,
):
""" """
:param headers: 请求头部信息 :param headers: 请求头部信息
:param ua: User-Agent字符串 :param ua: User-Agent字符串
@@ -596,12 +801,18 @@ class AsyncRequestUtils:
:param accept_type: Accept头部信息,默认为 "application/json" :param accept_type: Accept头部信息,默认为 "application/json"
:param verify: 是否校验证书 :param verify: 是否校验证书
:param follow_redirects: 客户端默认是否跟随重定向 :param follow_redirects: 客户端默认是否跟随重定向
:param max_keepalive_connections: 共享 AsyncHTTPTransport 的最大 keep-alive 连接数
:param max_connections: 共享 AsyncHTTPTransport 的最大连接数
:param keepalive_expiry: 共享 AsyncHTTPTransport 的 keep-alive 连接过期时间(秒)
""" """
self._proxies = self._convert_proxies_for_httpx(proxies) self._proxies = self._convert_proxies_for_httpx(proxies)
self._client = client self._client = client
self._timeout = timeout or 20 self._timeout = timeout or 20
self._verify = verify self._verify = verify
self._follow_redirects = follow_redirects self._follow_redirects = follow_redirects
self._max_keepalive_connections = max_keepalive_connections
self._max_connections = max_connections
self._keepalive_expiry = keepalive_expiry
if not content_type: if not content_type:
content_type = "application/x-www-form-urlencoded; charset=UTF-8" content_type = "application/x-www-form-urlencoded; charset=UTF-8"
if headers: if headers:
@@ -672,7 +883,9 @@ class AsyncRequestUtils:
except Exception as e: except Exception as e:
logger.debug(f"关闭异步响应失败: {e}") logger.debug(f"关闭异步响应失败: {e}")
async def request(self, method: str, url: str, raise_exception: bool = False, **kwargs) -> Optional[httpx.Response]: async def request(
self, method: str, url: str, raise_exception: bool = False, **kwargs
) -> Optional[httpx.Response]:
""" """
发起异步HTTP请求 发起异步HTTP请求
:param method: HTTP方法,如 get, post, put 等 :param method: HTTP方法,如 get, post, put 等
@@ -682,33 +895,104 @@ class AsyncRequestUtils:
:return: HTTP响应对象 :return: HTTP响应对象
:raises: httpx.RequestError 仅raise_exception为True时会抛出 :raises: httpx.RequestError 仅raise_exception为True时会抛出
""" """
if self._client is None: # 运行时 self._cookies 只能是 dict | Nonecookie_parse 默认 array=False 返回 dict
# 创建临时客户端 cookies_dict: Optional[dict] = self._cookies if isinstance(self._cookies, dict) else None
if self._client is not None:
# 用户自管 client 时,把实例级 cookies 注入到本次 per-request kwargs
# 既能复用用户 client,又不让 instance cookies 被静默丢弃。
# 调用方若显式传 kwargs["cookies"],则以其为准(setdefault 不覆盖)。
if cookies_dict is not None:
kwargs.setdefault("cookies", cookies_dict)
return await self._make_request(
self._client, method, url, raise_exception, **kwargs
)
# 共享底层 transport(连接池+TLS 复用),每次请求创建轻量 AsyncClient。
# AsyncClient 持有的 cookie jar 仅存活于本次请求 lifecycle
# 既复用握手又彻底避免 jar 跨调用累积。
transport = _get_shared_async_transport(
proxy=self._proxies,
verify=self._verify,
max_keepalive_connections=self._max_keepalive_connections,
max_connections=self._max_connections,
keepalive_expiry=self._keepalive_expiry,
)
if transport is not None:
# 用 _NonClosingTransportProxy 包装共享 transport,吞掉 AsyncClient.__aexit__
# 传播下来的 transport.__aexit__,避免每次 async with 退出都把共享连接池清空。
async with httpx.AsyncClient(
transport=_NonClosingTransportProxy(transport),
timeout=httpx.Timeout(self._timeout),
follow_redirects=self._follow_redirects,
cookies=cookies_dict,
) as client:
return await self._make_request(
client, method, url, raise_exception, **kwargs
)
# 兜底:没有运行中的事件循环时,临时客户端走完即关
async with httpx.AsyncClient( async with httpx.AsyncClient(
proxy=self._proxies, proxy=self._proxies,
timeout=self._timeout, timeout=self._timeout,
verify=self._verify, verify=self._verify,
follow_redirects=self._follow_redirects, follow_redirects=self._follow_redirects,
cookies=self._cookies # 在创建客户端时传入Cookie cookies=cookies_dict,
) as client: ) as client:
return await self._make_request(client, method, url, raise_exception, **kwargs) return await self._make_request(
else: client, method, url, raise_exception, **kwargs
return await self._make_request(self._client, method, url, raise_exception, **kwargs) )
async def _make_request(self, client: httpx.AsyncClient, method: str, url: str, raise_exception: bool = False, async def _make_request(
**kwargs) -> Optional[httpx.Response]: self,
client: httpx.AsyncClient,
method: str,
url: str,
raise_exception: bool = False,
**kwargs,
) -> Optional[httpx.Response]:
""" """
执行实际的异步请求 执行实际的异步请求
""" """
kwargs.setdefault("headers", self._headers) kwargs.setdefault("headers", self._headers)
# Cookie已经在AsyncClient创建时设置,不要在request时再设置,否则会被覆盖 # 共享池下 client 自带默认 timeout,这里用每请求 timeout 覆盖以尊重实例配置
# kwargs.setdefault("cookies", self._cookies) kwargs.setdefault("timeout", self._timeout)
# Cookie 在 request() 入口已按 path 处理:
# - path A(用户自管 client):kwargs["cookies"] 已注入
# - path B/C(新建 AsyncClient):构造时已绑定 cookies
# 这里不重复 setdefault,避免覆盖各 path 的设定
method_upper = method.upper()
# 仅对幂等方法做 stale-pool 竞态重试:复用了刚被对端 FIN 的 keep-alive 连接时,
# 实际请求通常未到服务端,httpx 自身不重试,这里兜底一次。
is_idempotent = method_upper in ("GET", "HEAD", "OPTIONS")
stale_conn_errs = (httpx.RemoteProtocolError, httpx.ReadError, httpx.WriteError)
try: try:
return await client.request(method, url, **kwargs) return await client.request(method, url, **kwargs)
except stale_conn_errs as e:
if is_idempotent:
logger.debug(f"keep-alive 连接已失效,幂等方法重试一次: {e!r}")
try:
return await client.request(method, url, **kwargs)
except httpx.RequestError as e2:
error_msg = (
str(e2) or f"未知网络错误 (URL: {url}, Method: {method_upper})"
)
logger.debug(f"重试后异步请求仍失败: {error_msg}")
if raise_exception:
raise
return None
# 非幂等方法(POST/PUT/PATCH/DELETE 等)不重试以避免重复副作用,
# 但仍记录调试日志,避免静默失败掩盖问题
error_msg = str(e) or f"未知网络错误 (URL: {url}, Method: {method_upper})"
logger.debug(f"异步请求失败(非幂等不重试): {error_msg}")
if raise_exception:
raise
return None
except httpx.RequestError as e: except httpx.RequestError as e:
# 获取更详细的错误信息 # 获取更详细的错误信息
error_msg = str(e) if str(e) else f"未知网络错误 (URL: {url}, Method: {method.upper()})" error_msg = str(e) or f"未知网络错误 (URL: {url}, Method: {method_upper})"
logger.debug(f"异步请求失败: {error_msg}") logger.debug(f"异步请求失败: {error_msg}")
if raise_exception: if raise_exception:
raise raise
@@ -736,7 +1020,9 @@ class AsyncRequestUtils:
if response is not None: if response is not None:
await response.aclose() await response.aclose()
async def post(self, url: str, data: Any = None, json: dict = None, **kwargs) -> Optional[httpx.Response]: async def post(
self, url: str, data: Any = None, json: dict = None, **kwargs
) -> Optional[httpx.Response]:
""" """
发送异步POST请求 发送异步POST请求
:param url: 请求的URL :param url: 请求的URL
@@ -745,9 +1031,13 @@ class AsyncRequestUtils:
:param kwargs: 其他请求参数,如headers, cookies, proxies等 :param kwargs: 其他请求参数,如headers, cookies, proxies等
:return: HTTP响应对象,若发生RequestError则返回None :return: HTTP响应对象,若发生RequestError则返回None
""" """
return await self.request(method="post", url=url, data=data, json=json, **kwargs) return await self.request(
method="post", url=url, data=data, json=json, **kwargs
)
async def put(self, url: str, data: Any = None, **kwargs) -> Optional[httpx.Response]: async def put(
self, url: str, data: Any = None, **kwargs
) -> Optional[httpx.Response]:
""" """
发送异步PUT请求 发送异步PUT请求
:param url: 请求的URL :param url: 请求的URL
@@ -757,14 +1047,16 @@ class AsyncRequestUtils:
""" """
return await self.request(method="put", url=url, data=data, **kwargs) return await self.request(method="put", url=url, data=data, **kwargs)
async def get_res(self, async def get_res(
self,
url: str, url: str,
params: dict = None, params: dict = None,
data: Any = None, data: Any = None,
json: dict = None, json: dict = None,
allow_redirects: bool = True, allow_redirects: bool = True,
raise_exception: bool = False, raise_exception: bool = False,
**kwargs) -> Optional[httpx.Response]: **kwargs,
) -> Optional[httpx.Response]:
""" """
发送异步GET请求并返回响应对象 发送异步GET请求并返回响应对象
:param url: 请求的URL :param url: 请求的URL
@@ -777,32 +1069,105 @@ class AsyncRequestUtils:
:return: HTTP响应对象,若发生RequestError则返回None :return: HTTP响应对象,若发生RequestError则返回None
:raises: httpx.RequestError 仅raise_exception为True时会抛出 :raises: httpx.RequestError 仅raise_exception为True时会抛出
""" """
return await self.request(method="get", return await self.request(
method="get",
url=url, url=url,
params=params, params=params,
data=data, data=data,
json=json, json=json,
follow_redirects=allow_redirects, follow_redirects=allow_redirects,
raise_exception=raise_exception, raise_exception=raise_exception,
**kwargs) **kwargs,
)
@asynccontextmanager @asynccontextmanager
async def get_stream(self, url: str, params: dict = None, **kwargs): async def get_stream(
self,
url: str,
params: dict = None,
raise_exception: bool = False,
**kwargs,
):
""" """
获取异步流式响应的上下文管理器,适用于大文件下载 获取异步流式响应的上下文管理器,适用于大文件下载
使用 httpx.AsyncClient.stream() 标准流式 API,避免把响应体一次性读入内存。
:param url: 请求的URL :param url: 请求的URL
:param params: 请求的参数 :param params: 请求的参数
:param kwargs: 其他请求参数 :param raise_exception: 是否在发生异常时抛出,否则吞掉并 yield None
:param kwargs: 其他请求参数(headers, cookies 等)
:return: 上下文管理器,进入后 yield httpx.Response(出错时 yield None
""" """
kwargs['stream'] = True cookies_dict: Optional[dict] = self._cookies if isinstance(self._cookies, dict) else None
response = await self.request(method="get", url=url, params=params, **kwargs) kwargs.setdefault("headers", self._headers)
try:
yield response
finally:
if response is not None:
await response.aclose()
async def post_res(self, # 与 _make_request 保持一致:复用 keep-alive 时偶遇对端 FIN 的连接,
# 流式 GET 是幂等的,单次重试即可
stale_conn_errs = (httpx.RemoteProtocolError, httpx.ReadError, httpx.WriteError)
async with AsyncExitStack() as stack:
# 选 client:复用与 request() 相同的三条 path 逻辑
if self._client is not None:
client = self._client
if cookies_dict is not None:
kwargs.setdefault("cookies", cookies_dict)
else:
transport = _get_shared_async_transport(
proxy=self._proxies,
verify=self._verify,
max_keepalive_connections=self._max_keepalive_connections,
max_connections=self._max_connections,
keepalive_expiry=self._keepalive_expiry,
)
if transport is not None:
client = await stack.enter_async_context(
httpx.AsyncClient(
transport=_NonClosingTransportProxy(transport),
timeout=httpx.Timeout(self._timeout),
follow_redirects=self._follow_redirects,
cookies=cookies_dict,
)
)
else:
client = await stack.enter_async_context(
httpx.AsyncClient(
proxy=self._proxies,
timeout=self._timeout,
verify=self._verify,
follow_redirects=self._follow_redirects,
cookies=cookies_dict,
)
)
try:
response = await stack.enter_async_context(
client.stream("GET", url, params=params, **kwargs)
)
except stale_conn_errs as e:
logger.debug(f"流式 keep-alive 连接已失效,重试一次: {e!r}")
try:
response = await stack.enter_async_context(
client.stream("GET", url, params=params, **kwargs)
)
except httpx.RequestError as e2:
logger.debug(f"重试后异步流式请求仍失败: {e2!r}")
if raise_exception:
raise
yield None
return
except httpx.RequestError as e:
logger.debug(f"异步流式请求失败: {e!r}")
if raise_exception:
raise
yield None
return
# AsyncExitStack 反向 unwind:先关 stream,再关 owned client
# yield 体内的异常由标准 async with 协议透传给各 __aexit__
yield response
async def post_res(
self,
url: str, url: str,
data: Any = None, data: Any = None,
params: dict = None, params: dict = None,
@@ -810,7 +1175,8 @@ class AsyncRequestUtils:
files: Any = None, files: Any = None,
json: dict = None, json: dict = None,
raise_exception: bool = False, raise_exception: bool = False,
**kwargs) -> Optional[httpx.Response]: **kwargs,
) -> Optional[httpx.Response]:
""" """
发送异步POST请求并返回响应对象 发送异步POST请求并返回响应对象
:param url: 请求的URL :param url: 请求的URL
@@ -824,7 +1190,8 @@ class AsyncRequestUtils:
:return: HTTP响应对象,若发生RequestError则返回None :return: HTTP响应对象,若发生RequestError则返回None
:raises: httpx.RequestError 仅raise_exception为True时会抛出 :raises: httpx.RequestError 仅raise_exception为True时会抛出
""" """
return await self.request(method="post", return await self.request(
method="post",
url=url, url=url,
data=data, data=data,
params=params, params=params,
@@ -832,9 +1199,11 @@ class AsyncRequestUtils:
files=files, files=files,
json=json, json=json,
raise_exception=raise_exception, raise_exception=raise_exception,
**kwargs) **kwargs,
)
async def put_res(self, async def put_res(
self,
url: str, url: str,
data: Any = None, data: Any = None,
params: dict = None, params: dict = None,
@@ -842,7 +1211,8 @@ class AsyncRequestUtils:
files: Any = None, files: Any = None,
json: dict = None, json: dict = None,
raise_exception: bool = False, raise_exception: bool = False,
**kwargs) -> Optional[httpx.Response]: **kwargs,
) -> Optional[httpx.Response]:
""" """
发送异步PUT请求并返回响应对象 发送异步PUT请求并返回响应对象
:param url: 请求的URL :param url: 请求的URL
@@ -856,7 +1226,8 @@ class AsyncRequestUtils:
:return: HTTP响应对象,若发生RequestError则返回None :return: HTTP响应对象,若发生RequestError则返回None
:raises: httpx.RequestError 仅raise_exception为True时会抛出 :raises: httpx.RequestError 仅raise_exception为True时会抛出
""" """
return await self.request(method="put", return await self.request(
method="put",
url=url, url=url,
data=data, data=data,
params=params, params=params,
@@ -864,15 +1235,18 @@ class AsyncRequestUtils:
files=files, files=files,
json=json, json=json,
raise_exception=raise_exception, raise_exception=raise_exception,
**kwargs) **kwargs,
)
async def delete_res(self, async def delete_res(
self,
url: str, url: str,
data: Any = None, data: Any = None,
params: dict = None, params: dict = None,
allow_redirects: bool = True, allow_redirects: bool = True,
raise_exception: bool = False, raise_exception: bool = False,
**kwargs) -> Optional[httpx.Response]: **kwargs,
) -> Optional[httpx.Response]:
""" """
发送异步DELETE请求并返回响应对象 发送异步DELETE请求并返回响应对象
:param url: 请求的URL :param url: 请求的URL
@@ -884,13 +1258,15 @@ class AsyncRequestUtils:
:return: HTTP响应对象,若发生RequestError则返回None :return: HTTP响应对象,若发生RequestError则返回None
:raises: httpx.RequestError 仅raise_exception为True时会抛出 :raises: httpx.RequestError 仅raise_exception为True时会抛出
""" """
return await self.request(method="delete", return await self.request(
method="delete",
url=url, url=url,
data=data, data=data,
params=params, params=params,
follow_redirects=allow_redirects, follow_redirects=allow_redirects,
raise_exception=raise_exception, raise_exception=raise_exception,
**kwargs) **kwargs,
)
async def get_json(self, url: str, params: dict = None, **kwargs) -> Optional[dict]: async def get_json(self, url: str, params: dict = None, **kwargs) -> Optional[dict]:
""" """
@@ -914,7 +1290,9 @@ class AsyncRequestUtils:
if response is not None: if response is not None:
await response.aclose() await response.aclose()
async def post_json(self, url: str, data: Any = None, json: dict = None, **kwargs) -> Optional[dict]: async def post_json(
self, url: str, data: Any = None, json: dict = None, **kwargs
) -> Optional[dict]:
""" """
发送异步POST请求并返回JSON数据,自动关闭连接 发送异步POST请求并返回JSON数据,自动关闭连接
:param url: 请求的URL :param url: 请求的URL
@@ -925,7 +1303,9 @@ class AsyncRequestUtils:
""" """
if json is None: if json is None:
json = {} json = {}
response = await self.request(method="post", url=url, data=data, json=json, **kwargs) response = await self.request(
method="post", url=url, data=data, json=json, **kwargs
)
try: try:
if response: if response:
try: try: