135 lines
4.0 KiB
Python
135 lines
4.0 KiB
Python
import logging
|
|
import os
|
|
import time
|
|
from collections import deque
|
|
from dataclasses import dataclass, field
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
HISTORY_CAPACITY = 60
|
|
|
|
|
|
@dataclass
|
|
class _MetricPoint:
|
|
ts: float
|
|
value: float
|
|
labels: dict[str, str] = field(default_factory=dict)
|
|
|
|
|
|
class _MetricFamily:
|
|
def __init__(self, name: str, help_text: str, metric_type: str):
|
|
self.name = name
|
|
self.help = help_text
|
|
self.type = metric_type
|
|
self._buffer: deque[_MetricPoint] = deque(maxlen=HISTORY_CAPACITY)
|
|
self._last_value: float | None = None
|
|
self._last_labels: dict[str, str] | None = None
|
|
|
|
def set(self, value: float, labels: dict[str, str] | None = None):
|
|
self._last_value = value
|
|
self._last_labels = labels
|
|
self._buffer.append(_MetricPoint(ts=time.time(), value=value, labels=labels or {}))
|
|
|
|
@property
|
|
def value(self) -> float | None:
|
|
return self._last_value
|
|
|
|
def history(self) -> list[dict]:
|
|
return [{"ts": p.ts, "value": p.value, "labels": p.labels} for p in self._buffer]
|
|
|
|
def prometheus_line(self) -> str:
|
|
if self._last_value is None:
|
|
return ""
|
|
labels = self._last_labels or {}
|
|
label_str = "{" + ",".join(f'{k}="{v}"' for k, v in labels.items()) + "}" if labels else ""
|
|
return f"{self.name}{label_str} {self._last_value}\n"
|
|
|
|
|
|
class MetricsRegistry:
|
|
def __init__(self):
|
|
self._metrics: dict[str, _MetricFamily] = {}
|
|
|
|
def gauge(self, name: str, help_text: str) -> _MetricFamily:
|
|
if name not in self._metrics:
|
|
self._metrics[name] = _MetricFamily(name, help_text, "gauge")
|
|
return self._metrics[name]
|
|
|
|
def collect_prometheus(self) -> str:
|
|
lines = []
|
|
seen_help: set[str] = set()
|
|
for m in self._metrics.values():
|
|
if m.name not in seen_help:
|
|
lines.append(f"# HELP {m.name} {m.help}")
|
|
lines.append(f"# TYPE {m.name} {m.type}")
|
|
seen_help.add(m.name)
|
|
line = m.prometheus_line()
|
|
if line:
|
|
lines.append(line)
|
|
return "\n".join(lines) + "\n"
|
|
|
|
def history(self, name: str) -> list[dict]:
|
|
m = self._metrics.get(name)
|
|
return m.history() if m else []
|
|
|
|
def all_history(self) -> dict[str, list[dict]]:
|
|
return {name: m.history() for name, m in self._metrics.items()}
|
|
|
|
|
|
metrics_registry = MetricsRegistry()
|
|
|
|
event_loop_delay_p99 = metrics_registry.gauge(
|
|
"yuxi_event_loop_delay_p99_ms",
|
|
"Event loop P99 delay in milliseconds",
|
|
)
|
|
event_loop_utilization = metrics_registry.gauge(
|
|
"yuxi_event_loop_utilization",
|
|
"Event loop utilization ratio",
|
|
)
|
|
memory_rss_mb = metrics_registry.gauge(
|
|
"yuxi_memory_rss_mb",
|
|
"Process RSS memory in MB",
|
|
)
|
|
memory_gc_objects = metrics_registry.gauge(
|
|
"yuxi_memory_gc_objects",
|
|
"Number of objects tracked by GC",
|
|
)
|
|
active_sessions = metrics_registry.gauge(
|
|
"yuxi_active_sessions",
|
|
"Number of active channel sessions",
|
|
)
|
|
stuck_sessions = metrics_registry.gauge(
|
|
"yuxi_stuck_sessions",
|
|
"Number of stuck channel sessions",
|
|
)
|
|
channel_health = metrics_registry.gauge(
|
|
"yuxi_channel_health",
|
|
"Channel health status (1=healthy, 0=unhealthy)",
|
|
)
|
|
active_tasks = metrics_registry.gauge(
|
|
"yuxi_active_tasks",
|
|
"Number of active asyncio tasks",
|
|
)
|
|
|
|
|
|
async def send_alert_webhook(title: str, details: dict | None = None):
|
|
webhook_url = os.environ.get("YUXI_ALERT_WEBHOOK_URL")
|
|
if not webhook_url:
|
|
return
|
|
|
|
payload = {
|
|
"title": title,
|
|
"timestamp": time.time(),
|
|
}
|
|
if details:
|
|
payload["details"] = details
|
|
|
|
try:
|
|
import aiohttp
|
|
|
|
async with aiohttp.ClientSession() as session:
|
|
async with session.post(webhook_url, json=payload, timeout=aiohttp.ClientTimeout(total=5)) as resp:
|
|
if resp.status >= 400:
|
|
logger.warning("Alert webhook failed: HTTP %s", resp.status)
|
|
except Exception:
|
|
logger.warning("Alert webhook delivery failed", exc_info=True)
|