Merge pull request #6587 from ga626/codex/github-transport-recovery

This commit is contained in:
jxxghp
2026-09-06 11:20:13 +08:00
committed by GitHub
3 changed files with 83 additions and 22 deletions
+28 -12
View File
@@ -50,6 +50,7 @@ PLUGIN_INDEX_MAX_ENTRIES = 4096
PLUGIN_INDEX_MAX_HISTORY_ENTRIES = 512 PLUGIN_INDEX_MAX_HISTORY_ENTRIES = 512
PLUGIN_INDEX_MAX_NESTING = 64 PLUGIN_INDEX_MAX_NESTING = 64
PLUGIN_INDEX_READ_CHUNK_SIZE = 64 * 1024 PLUGIN_INDEX_READ_CHUNK_SIZE = 64 * 1024
PLUGIN_INDEX_REQUEST_TIMEOUT = 15
PLUGIN_INDEX_COMPATIBILITY_FLAG_PATTERN = re.compile(r"^v\d+t?$") PLUGIN_INDEX_COMPATIBILITY_FLAG_PATTERN = re.compile(r"^v\d+t?$")
PLUGIN_INDEX_TEXT_LIMITS = { PLUGIN_INDEX_TEXT_LIMITS = {
"name": 256, "name": 256,
@@ -68,6 +69,12 @@ class _PluginIndexTooLargeError(RuntimeError):
"""插件索引的声明长度或实际读取字节超过资源边界。""" """插件索引的声明长度或实际读取字节超过资源边界。"""
def _format_request_error(error: Exception) -> str:
"""保留异常类型,避免无消息异常在运行日志中显示为空白。"""
detail = str(error).strip()
return f"{type(error).__name__}: {detail}" if detail else type(error).__name__
def build_local_repo_url( def build_local_repo_url(
plugin_id: str, plugin_id: str,
*, *,
@@ -903,7 +910,7 @@ class PluginMarketTransport(metaclass=WeakSingleton):
cls, cls,
url: str, url: str,
headers: Optional[dict[str, str]] = None, headers: Optional[dict[str, str]] = None,
timeout: Optional[int] = 60, timeout: Optional[int] = PLUGIN_INDEX_REQUEST_TIMEOUT,
) -> Optional[tuple[int, str]]: ) -> Optional[tuple[int, str]]:
"""按 GitHub 降级顺序流式读取同步插件索引,并限制解压后字节数。""" """按 GitHub 降级顺序流式读取同步插件索引,并限制解压后字节数。"""
strategies = cls._build_github_request_strategies( strategies = cls._build_github_request_strategies(
@@ -929,7 +936,7 @@ class PluginMarketTransport(metaclass=WeakSingleton):
except Exception as error: # noqa: BLE001 - 失败后尝试下一传输策略 except Exception as error: # noqa: BLE001 - 失败后尝试下一传输策略
logger.error( logger.error(
f"[GitHub] 插件索引请求失败,策略:{strategy_name}" f"[GitHub] 插件索引请求失败,策略:{strategy_name}"
f"URL{target_url},错误:{error}" f"URL{target_url},错误:{_format_request_error(error)}"
) )
logger.error(f"[GitHub] 所有策略均无法读取插件索引,URL:{url}") logger.error(f"[GitHub] 所有策略均无法读取插件索引,URL:{url}")
return None return None
@@ -939,7 +946,7 @@ class PluginMarketTransport(metaclass=WeakSingleton):
cls, cls,
url: str, url: str,
headers: Optional[dict[str, str]] = None, headers: Optional[dict[str, str]] = None,
timeout: Optional[int] = 60, timeout: Optional[int] = PLUGIN_INDEX_REQUEST_TIMEOUT,
) -> Optional[tuple[int, str]]: ) -> Optional[tuple[int, str]]:
"""按 GitHub 降级顺序流式读取异步插件索引,并限制解压后字节数。""" """按 GitHub 降级顺序流式读取异步插件索引,并限制解压后字节数。"""
strategies = cls._build_github_request_strategies( strategies = cls._build_github_request_strategies(
@@ -965,7 +972,7 @@ class PluginMarketTransport(metaclass=WeakSingleton):
except Exception as error: # noqa: BLE001 - 失败后尝试下一传输策略 except Exception as error: # noqa: BLE001 - 失败后尝试下一传输策略
logger.error( logger.error(
f"[GitHub] 插件索引请求失败,策略:{strategy_name}" f"[GitHub] 插件索引请求失败,策略:{strategy_name}"
f"URL{target_url},错误:{error}" f"URL{target_url},错误:{_format_request_error(error)}"
) )
logger.error(f"[GitHub] 所有策略均无法读取插件索引,URL:{url}") logger.error(f"[GitHub] 所有策略均无法读取插件索引,URL:{url}")
return None return None
@@ -1251,7 +1258,7 @@ class PluginMarketTransport(metaclass=WeakSingleton):
timeout: Optional[int] = 60, timeout: Optional[int] = 60,
is_api: bool = False, is_api: bool = False,
) -> list[tuple[str, str, PluginRequestOptions]]: ) -> list[tuple[str, str, PluginRequestOptions]]:
"""构造同步与异步 GitHub 请求共用的镜像、代理和直连顺序。""" """构造同步与异步 GitHub 请求共用的镜像和单一出口顺序。"""
strategies: list[tuple[str, str, PluginRequestOptions]] = [] strategies: list[tuple[str, str, PluginRequestOptions]] = []
if not is_api and get_runtime_setting('GITHUB_PROXY'): if not is_api and get_runtime_setting('GITHUB_PROXY'):
proxy_url = ( proxy_url = (
@@ -1272,9 +1279,10 @@ class PluginMarketTransport(metaclass=WeakSingleton):
}, },
) )
) )
strategies.append( else:
("直连", url, {"headers": headers, "timeout": timeout}) strategies.append(
) ("直连", url, {"headers": headers, "timeout": timeout})
)
return strategies return strategies
@staticmethod @staticmethod
@@ -1283,7 +1291,8 @@ class PluginMarketTransport(metaclass=WeakSingleton):
timeout: Optional[int] = 60, timeout: Optional[int] = 60,
is_api: bool = False) -> Optional[Response]: is_api: bool = False) -> Optional[Response]:
""" """
使用自动降级策略,请求资源,优先级依次为镜像站、代理、直连 使用自动降级策略,请求资源:可选镜像站后只使用一个出口;
显式配置代理时不再追加无效直连。
:param url: 目标URL :param url: 目标URL
:param headers: 请求头信息 :param headers: 请求头信息
:param timeout: 请求超时时间 :param timeout: 请求超时时间
@@ -1306,7 +1315,10 @@ class PluginMarketTransport(metaclass=WeakSingleton):
logger.debug(f"[GitHub] 请求成功,策略:{strategy_name}, URL: {target_url}") logger.debug(f"[GitHub] 请求成功,策略:{strategy_name}, URL: {target_url}")
return res return res
except Exception as e: except Exception as e:
logger.error(f"[GitHub] 请求失败,策略:{strategy_name}, URL: {target_url},错误:{str(e)}") logger.error(
f"[GitHub] 请求失败,策略:{strategy_name}, URL: {target_url}"
f"错误:{_format_request_error(e)}"
)
logger.error(f"[GitHub] 所有策略均请求失败,URL: {url},请检查网络连接或 GitHub 配置") logger.error(f"[GitHub] 所有策略均请求失败,URL: {url},请检查网络连接或 GitHub 配置")
return None return None
@@ -1404,7 +1416,8 @@ class PluginMarketTransport(metaclass=WeakSingleton):
timeout: Optional[int] = 60, timeout: Optional[int] = 60,
is_api: bool = False) -> Optional[httpx2.Response]: is_api: bool = False) -> Optional[httpx2.Response]:
""" """
使用自动降级策略,异步请求资源,优先级依次为镜像站、代理、直连 使用自动降级策略,异步请求资源:可选镜像站后只使用一个出口;
显式配置代理时不再追加无效直连。
:param url: 目标URL :param url: 目标URL
:param headers: 请求头信息 :param headers: 请求头信息
:param timeout: 请求超时时间 :param timeout: 请求超时时间
@@ -1427,7 +1440,10 @@ class PluginMarketTransport(metaclass=WeakSingleton):
logger.debug(f"[GitHub] 请求成功,策略:{strategy_name}, URL: {target_url}") logger.debug(f"[GitHub] 请求成功,策略:{strategy_name}, URL: {target_url}")
return res return res
except Exception as e: except Exception as e:
logger.error(f"[GitHub] 请求失败,策略:{strategy_name}, URL: {target_url},错误:{str(e)}") logger.error(
f"[GitHub] 请求失败,策略:{strategy_name}, URL: {target_url}"
f"错误:{_format_request_error(e)}"
)
logger.error(f"[GitHub] 所有策略均请求失败,URL: {url},请检查网络连接或 GitHub 配置") logger.error(f"[GitHub] 所有策略均请求失败,URL: {url},请检查网络连接或 GitHub 配置")
return None return None
+49
View File
@@ -6,16 +6,65 @@ from contextlib import asynccontextmanager, contextmanager
import pytest import pytest
from app.adapters.external.plugin.client import ( from app.adapters.external.plugin.client import (
PLUGIN_INDEX_REQUEST_TIMEOUT,
PLUGIN_INDEX_MAX_BYTES, PLUGIN_INDEX_MAX_BYTES,
PLUGIN_INDEX_MAX_ENTRIES, PLUGIN_INDEX_MAX_ENTRIES,
PluginMarketClient, PluginMarketClient,
PluginMarketTransport, PluginMarketTransport,
_format_request_error,
) )
SYNC_INDEX_REQUEST = "_PluginMarketTransport__request_plugin_index_with_fallback" SYNC_INDEX_REQUEST = "_PluginMarketTransport__request_plugin_index_with_fallback"
ASYNC_INDEX_REQUEST = "_PluginMarketTransport__async_request_plugin_index_with_fallback" ASYNC_INDEX_REQUEST = "_PluginMarketTransport__async_request_plugin_index_with_fallback"
def test_plugin_index_request_timeout_is_bounded() -> None:
"""插件索引故障应在有限时间内返回,不能沿用一分钟等待。"""
assert PLUGIN_INDEX_REQUEST_TIMEOUT == 15
def test_github_request_strategies_do_not_retry_direct_when_proxy_is_configured(
monkeypatch,
) -> None:
"""显式代理失败后不得绕过同一出口策略再等待一次直连超时。"""
settings = {
"GITHUB_PROXY": None,
"PROXY_HOST": "http://proxy:7891",
"PROXY": {"http": "http://proxy:7891", "https": "http://proxy:7891"},
}
monkeypatch.setattr(
"app.adapters.external.plugin.client.get_runtime_setting",
lambda key: settings.get(key),
)
strategies = PluginMarketTransport._build_github_request_strategies(
url="https://raw.githubusercontent.com/example/repo/main/package.v3.json",
)
assert [name for name, _url, _params in strategies] == ["代理"]
def test_github_request_strategies_use_direct_without_configured_proxy(
monkeypatch,
) -> None:
"""未配置代理时仍保留直连出口。"""
monkeypatch.setattr(
"app.adapters.external.plugin.client.get_runtime_setting",
lambda _key: None,
)
strategies = PluginMarketTransport._build_github_request_strategies(
url="https://raw.githubusercontent.com/example/repo/main/package.v3.json",
)
assert [name for name, _url, _params in strategies] == ["直连"]
def test_request_error_formatter_keeps_type_for_blank_exception() -> None:
"""无消息异常也必须给运行日志留下可诊断类型。"""
assert _format_request_error(TimeoutError()) == "TimeoutError"
class _SyncStreamResponse: class _SyncStreamResponse:
"""提供 requests 流式响应所需的最小测试契约。""" """提供 requests 流式响应所需的最小测试契约。"""
+6 -10
View File
@@ -12,7 +12,7 @@ from app.adapters.external.plugin.client import PluginMarketTransport
async def test_sync_and_async_github_requests_share_fallback_policy( async def test_sync_and_async_github_requests_share_fallback_policy(
monkeypatch, monkeypatch,
) -> None: ) -> None:
"""同步与异步请求必须使用相同镜像、代理、直连顺序参数。""" """同步与异步请求必须使用相同镜像和单一出口顺序参数。"""
proxy = {"all": "http://proxy.example:7890"} proxy = {"all": "http://proxy.example:7890"}
runtime_settings = SimpleNamespace( runtime_settings = SimpleNamespace(
GITHUB_PROXY="https://mirror.example", GITHUB_PROXY="https://mirror.example",
@@ -29,16 +29,16 @@ async def test_sync_and_async_github_requests_share_fallback_policy(
response = object() response = object()
class SyncRequest: class SyncRequest:
"""记录同步请求,并让前两种策略失败以遍历完整顺序""" """记录同步请求,并让镜像策略失败以验证代理接管"""
def __init__(self, **kwargs) -> None: def __init__(self, **kwargs) -> None:
self._kwargs = kwargs self._kwargs = kwargs
def get_res(self, *, url: str, raise_exception: bool): def get_res(self, *, url: str, raise_exception: bool):
"""记录请求目标,第次返回固定响应。""" """记录请求目标,第次返回固定响应。"""
assert raise_exception is True assert raise_exception is True
sync_requests.append((self._kwargs, url)) sync_requests.append((self._kwargs, url))
if len(sync_requests) < 3: if len(sync_requests) < 2:
raise RuntimeError("next strategy") raise RuntimeError("next strategy")
return response return response
@@ -49,10 +49,10 @@ async def test_sync_and_async_github_requests_share_fallback_policy(
self._kwargs = kwargs self._kwargs = kwargs
async def get_res(self, *, url: str, raise_exception: bool): async def get_res(self, *, url: str, raise_exception: bool):
"""记录请求目标,第次返回固定响应。""" """记录请求目标,第次返回固定响应。"""
assert raise_exception is True assert raise_exception is True
async_requests.append((self._kwargs, url)) async_requests.append((self._kwargs, url))
if len(async_requests) < 3: if len(async_requests) < 2:
raise RuntimeError("next strategy") raise RuntimeError("next strategy")
return response return response
@@ -80,10 +80,6 @@ async def test_sync_and_async_github_requests_share_fallback_policy(
{"headers": {"X-Test": "1"}, "proxies": proxy, "timeout": 12}, {"headers": {"X-Test": "1"}, "proxies": proxy, "timeout": 12},
"https://api.example/resource", "https://api.example/resource",
), ),
(
{"headers": {"X-Test": "1"}, "timeout": 12},
"https://api.example/resource",
),
] ]