feat: add low-cardinality observation port

This commit is contained in:
jxxghp
2026-08-21 22:15:47 +08:00
parent 47f1ff9cb4
commit 43cccb8ff8
12 changed files with 454 additions and 24 deletions
+18 -7
View File
@@ -13,6 +13,7 @@ from app.runtime.event.registry import EventRegistry
from app.runtime.execution import run_in_threadpool
from app.runtime.log import logger
from app.runtime.correlation import correlation_scope
from app.runtime.observability import observe_duration
from app.schemas.types import EventType
@@ -147,7 +148,12 @@ class EventDispatcher:
method, binding, class_name, method_name = resolved
with correlation_scope(event.correlation_id):
try:
method(event)
with observe_duration(
"event.handler.duration",
event_type=event.event_type.value,
handler_type="bound" if class_name else "function",
):
method(event)
except Exception as err:
self._error_handler(
event=event,
@@ -165,12 +171,17 @@ class EventDispatcher:
method, binding, class_name, method_name = resolved
with correlation_scope(event.correlation_id):
try:
if inspect.iscoroutinefunction(method):
await method(event)
elif binding.run_sync_in_threadpool or not class_name:
await run_in_threadpool(method, event)
else:
method(event)
with observe_duration(
"event.handler.duration",
event_type=event.event_type.value,
handler_type="bound" if class_name else "function",
):
if inspect.iscoroutinefunction(method):
await method(event)
elif binding.run_sync_in_threadpool or not class_name:
await run_in_threadpool(method, event)
else:
method(event)
except Exception as err:
self._error_handler(
event=event,
+11
View File
@@ -22,6 +22,7 @@ from app.runtime.event.errors import EventErrorNotifier, EventErrorPolicy
from app.runtime.event.registry import EventRegistry
from app.runtime.event.contracts import validate_event_payload
from app.runtime.correlation import get_correlation_id
from app.runtime.observability import record_metric
DEFAULT_EVENT_PRIORITY = 10 # 事件的默认优先级
MIN_EVENT_CONSUMER_THREADS = 1 # 最小事件消费者线程数
@@ -317,6 +318,11 @@ class EventManager(metaclass=Singleton):
"""
logger.debug(f"Triggering broadcast event: {event}")
self.__event_queue.put((event.priority, event))
record_metric(
"event.queue.depth",
self.__event_queue.qsize(),
delivery="broadcast",
)
def __dispatch_chain_event(self, event: Event) -> bool:
"""
@@ -421,6 +427,11 @@ class EventManager(metaclass=Singleton):
while self.__event.is_set():
try:
priority, event = self.__event_queue.get(timeout=rate_limiter.current_wait)
record_metric(
"event.queue.depth",
self.__event_queue.qsize(),
delivery="broadcast",
)
rate_limiter.reset()
self.__dispatch_broadcast_event(event)
except Empty:
+27 -14
View File
@@ -9,6 +9,7 @@ from typing import Any, Protocol
from app.foundation.reflection import ObjectUtils
from app.runtime.execution import run_in_threadpool
from app.runtime.log import logger
from app.runtime.observability import observe_duration
from app.runtime.extensions.module.contracts import (
diagnose_module_callable,
get_module_method_contract,
@@ -68,29 +69,41 @@ class ModuleInvocationDispatcher:
"""先执行插件模块,再按优先级执行宿主模块。"""
contract = get_module_method_contract(method)
logger.debug("模块方法契约:%s -> %s", method, contract.family)
result = self.execute_plugin_modules(method, None, *args, **kwargs)
with observe_duration(
"module.provider.duration", method=method, provider_type="plugin"
):
result = self.execute_plugin_modules(method, None, *args, **kwargs)
if not self.is_valid_empty(result) and not isinstance(result, list):
return result
return self.execute_system_modules(method, result, *args, **kwargs)
with observe_duration(
"module.provider.duration", method=method, provider_type="system"
):
return self.execute_system_modules(method, result, *args, **kwargs)
async def async_dispatch(self, method: str, *args: Any, **kwargs: Any) -> Any:
"""以与同步路径相同的聚合规则执行同步或异步模块方法。"""
contract = get_module_method_contract(method)
logger.debug("异步模块方法契约:%s -> %s", method, contract.family)
result = await self.async_execute_plugin_modules(
method,
None,
*args,
**kwargs,
)
with observe_duration(
"module.provider.duration", method=method, provider_type="plugin"
):
result = await self.async_execute_plugin_modules(
method,
None,
*args,
**kwargs,
)
if not self.is_valid_empty(result) and not isinstance(result, list):
return result
return await self.async_execute_system_modules(
method,
result,
*args,
**kwargs,
)
with observe_duration(
"module.provider.duration", method=method, provider_type="system"
):
return await self.async_execute_system_modules(
method,
result,
*args,
**kwargs,
)
def execute_plugin_modules(
self,
+99
View File
@@ -0,0 +1,99 @@
"""低基数运行指标端口与进程级 Facade。"""
from __future__ import annotations
import time
from contextlib import contextmanager
from dataclasses import dataclass
from enum import StrEnum
from typing import Iterator, Mapping, Protocol
class MetricKind(StrEnum):
"""声明指标采用 counter、histogram 或 gauge 语义。"""
COUNTER = "counter"
HISTOGRAM = "histogram"
GAUGE = "gauge"
@dataclass(frozen=True, slots=True)
class MetricSpec:
"""声明稳定指标名称、类型与允许的低基数标签。"""
name: str
kind: MetricKind
labels: frozenset[str]
METRIC_SPECS = {
spec.name: spec
for spec in (
MetricSpec("http.server.duration", MetricKind.HISTOGRAM, frozenset({"route", "method", "status"})),
MetricSpec("db.pool.wait", MetricKind.HISTOGRAM, frozenset({"backend", "outcome"})),
MetricSpec("db.pool.checked_out", MetricKind.GAUGE, frozenset({"backend"})),
MetricSpec("db.pool.timeout", MetricKind.COUNTER, frozenset({"backend"})),
MetricSpec("event.queue.depth", MetricKind.GAUGE, frozenset({"delivery"})),
MetricSpec("event.handler.duration", MetricKind.HISTOGRAM, frozenset({"event_type", "handler_type", "outcome"})),
MetricSpec("module.provider.duration", MetricKind.HISTOGRAM, frozenset({"method", "provider_type", "outcome"})),
MetricSpec("module.provider.timeout", MetricKind.COUNTER, frozenset({"method", "provider_type"})),
MetricSpec("scheduler.job.duration", MetricKind.HISTOGRAM, frozenset({"owner", "outcome"})),
MetricSpec("scheduler.job.overlap_skip", MetricKind.COUNTER, frozenset({"owner"})),
MetricSpec("scheduler.job.retry", MetricKind.COUNTER, frozenset({"owner"})),
MetricSpec("scheduler.job.dead_letter", MetricKind.COUNTER, frozenset({"owner"})),
MetricSpec("plugin.lifecycle.duration", MetricKind.HISTOGRAM, frozenset({"operation", "outcome"})),
MetricSpec("agent.active_tasks", MetricKind.GAUGE, frozenset({"task_type"})),
MetricSpec("agent.cancel", MetricKind.COUNTER, frozenset({"task_type", "outcome"})),
MetricSpec("agent.provider.duration", MetricKind.HISTOGRAM, frozenset({"provider_type", "outcome"})),
MetricSpec("agent.token_usage", MetricKind.COUNTER, frozenset({"provider_type", "direction"})),
)
}
class ObservationPort(Protocol):
"""Application/runtime 可依赖的最小指标写入端口。"""
def record(self, spec: MetricSpec, value: float, labels: Mapping[str, str]) -> None:
"""记录一个已经通过标签合同校验的指标值。"""
class NoopObservationPort:
"""未安装或未启用 exporter 时完全无副作用的默认实现。"""
def record(self, spec: MetricSpec, value: float, labels: Mapping[str, str]) -> None:
"""接受合法指标但不分配 exporter 资源。"""
_observation_port: ObservationPort = NoopObservationPort()
def configure_observation(port: ObservationPort | None) -> None:
"""由组合根替换进程级端口;None 明确恢复 no-op。"""
global _observation_port
_observation_port = port or NoopObservationPort()
def record_metric(name: str, value: float = 1, **labels: str) -> None:
"""校验名称、类型无关数值和标签白名单后写入当前端口。"""
spec = METRIC_SPECS[name]
unexpected = set(labels) - spec.labels
if unexpected:
raise ValueError(f"指标 {name} 包含未登记标签:{sorted(unexpected)}")
_observation_port.record(spec, float(value), labels)
@contextmanager
def observe_duration(name: str, **labels: str) -> Iterator[None]:
"""记录代码块耗时,并把成功或失败收敛为低基数 outcome。"""
started_at = time.perf_counter()
outcome = "success"
try:
yield
except BaseException:
outcome = "error"
raise
finally:
duration_labels = dict(labels)
if "outcome" in METRIC_SPECS[name].labels:
duration_labels["outcome"] = outcome
record_metric(name, time.perf_counter() - started_at, **duration_labels)