Kris
ed2ee086de
主要变更： 1. 新增UIActionScheduler实现优先级UI任务调度，支持CRITICAL/HIGH/NORMAL/LOW四级优先级 2. 替换原有SendQueue为UIActionScheduler，统一所有UI操作的调度逻辑 3. 为截图、登录、重启等API添加调度器封装，支持超时、限流与fast-fail机制 4. 新增Prometheus监控指标，统计UI任务执行、等待耗时、超时、限流与快速失败情况 5. 为QrCapture添加命令执行超时保护，避免X11操作卡死 6. 扩展StatusResponse与状态接口，暴露调度器指标 7. 新增完整的单元测试覆盖调度器功能 8. 为看门狗注入调度器，实现kill前队列清空与快速失败窗口
197 lines
5.2 KiB
Python
197 lines
5.2 KiB
Python
"""Prometheus 指标定义：发送计数/耗时/失败/图像匹配/DB校验/队列/运行态/熔断/缓存/自适应等待。
|
||
|
||
所有指标用 prometheus_client 定义，通过 /metrics 端点暴露（app.py 挂载）。
|
||
|
||
prometheus_client 为可选依赖：缺失时本模块降级为 NoOp 桩件，所有指标调用
|
||
（.labels()/.inc()/.set()/.observe() 等）变为 no-op，bridge 核心业务不受影响。
|
||
app.py 的 /metrics 端点会同步降级（make_asgi_app 仍 ImportError → 禁用并 warning）。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
import time
|
||
|
||
logger = logging.getLogger("woc-bridge")
|
||
|
||
|
||
class _NoOpMetric:
|
||
"""prometheus_client 缺失时的 NoOp 桩件。
|
||
|
||
覆盖上层所有调用模式：实例化（接收任意参数）+
|
||
.labels(*args, **kwargs) → 返回 self，支持链式 .inc()/.dec()/.set()/.observe()。
|
||
所有操作 no-op，不持有任何状态。
|
||
"""
|
||
|
||
__slots__ = ()
|
||
|
||
def __init__(self, *args, **kwargs) -> None:
|
||
pass
|
||
|
||
def labels(self, *args, **kwargs) -> "_NoOpMetric":
|
||
return self
|
||
|
||
def inc(self, amount: float = 1.0) -> None:
|
||
pass
|
||
|
||
def dec(self, amount: float = 1.0) -> None:
|
||
pass
|
||
|
||
def set(self, value: float) -> None:
|
||
pass
|
||
|
||
def observe(self, amount: float) -> None:
|
||
pass
|
||
|
||
def info(self, val) -> None:
|
||
pass
|
||
|
||
|
||
try:
|
||
from prometheus_client import (
|
||
Counter,
|
||
Gauge,
|
||
Histogram,
|
||
Info,
|
||
)
|
||
_PROMETHEUS_AVAILABLE = True
|
||
except ImportError:
|
||
logger.warning(
|
||
"[metrics] prometheus_client 不可用，指标降级为 NoOp；"
|
||
"/metrics 端点将自动禁用，核心业务不受影响"
|
||
)
|
||
_PROMETHEUS_AVAILABLE = False
|
||
# 类型忽略：故意将多个指标类名重绑定为同一个 NoOp 类，
|
||
# 使所有 Counter/Gauge/Histogram/Info(...) 实例化返回 _NoOpMetric 实例
|
||
Counter = Gauge = Histogram = Info = _NoOpMetric # type: ignore[assignment,misc]
|
||
|
||
# 发送总数（按 flow_name / result 标签）
|
||
send_total = Counter(
|
||
"woc_send_total",
|
||
"Total send operations",
|
||
["flow_name", "result"], # result: success / failed / skipped
|
||
)
|
||
|
||
# 发送耗时分布
|
||
send_duration = Histogram(
|
||
"woc_send_duration_seconds",
|
||
"Send operation duration in seconds",
|
||
["flow_name"],
|
||
buckets=(0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0),
|
||
)
|
||
|
||
# 按失败状态分类的失败计数
|
||
send_failed_by_state = Counter(
|
||
"woc_send_failed_by_state_total",
|
||
"Send failures by flow state",
|
||
["flow_name", "state"],
|
||
)
|
||
|
||
# 图像匹配置信度分布
|
||
image_match_confidence = Histogram(
|
||
"woc_image_match_confidence",
|
||
"OpenCV template match confidence",
|
||
["kind"],
|
||
buckets=(0.5, 0.7, 0.8, 0.85, 0.9, 0.95, 1.0),
|
||
)
|
||
|
||
# DB 校验耗时
|
||
db_verify_duration = Histogram(
|
||
"woc_db_verify_duration_seconds",
|
||
"DB verify duration in seconds",
|
||
buckets=(0.1, 0.2, 0.5, 1.0, 2.0, 3.0, 5.0),
|
||
)
|
||
|
||
# 发送队列待处理数
|
||
send_queue_pending = Gauge(
|
||
"woc_send_queue_pending",
|
||
"Send queue pending count",
|
||
)
|
||
|
||
# WeChat 运行态（1=running, 0=not running）
|
||
wechat_running = Gauge(
|
||
"woc_wechat_running",
|
||
"WeChat process running state (1=running, 0=not)",
|
||
)
|
||
|
||
# 熔断器状态（0=closed, 1=open, 2=half_open）
|
||
circuit_state = Gauge(
|
||
"woc_circuit_state",
|
||
"Circuit breaker state (0=closed, 1=open, 2=half_open)",
|
||
["name"],
|
||
)
|
||
|
||
# 会话缓存命中
|
||
session_cache_hit = Counter(
|
||
"woc_session_cache_hit_total",
|
||
"Session cache hit count",
|
||
["result"], # hit / miss
|
||
)
|
||
|
||
# 自适应等待超时计数
|
||
adaptive_wait_timeout = Counter(
|
||
"woc_adaptive_wait_timeout_total",
|
||
"Adaptive wait timeout count",
|
||
["kind"],
|
||
)
|
||
|
||
# 首次发送耗时（无缓存）
|
||
send_duration_first = Histogram(
|
||
"woc_send_duration_first_seconds",
|
||
"First send duration (no session cache) in seconds",
|
||
buckets=(1.0, 3.0, 5.0, 10.0, 30.0, 60.0),
|
||
)
|
||
|
||
# 缓存命中后的发送耗时
|
||
send_duration_cached = Histogram(
|
||
"woc_send_duration_cached_seconds",
|
||
"Cached send duration (session cache hit) in seconds",
|
||
buckets=(0.5, 1.0, 2.0, 5.0, 10.0, 30.0),
|
||
)
|
||
|
||
|
||
# UIActionScheduler 调度器指标
|
||
ui_action_executed = Counter(
|
||
"woc_ui_action_executed_total",
|
||
"UI actions executed total",
|
||
["priority"], # CRITICAL / HIGH / NORMAL / LOW
|
||
)
|
||
|
||
ui_action_wait_duration = Histogram(
|
||
"woc_ui_action_wait_duration_seconds",
|
||
"UI action wait duration (queue wait time)",
|
||
buckets=(0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0),
|
||
)
|
||
|
||
ui_action_exec_duration = Histogram(
|
||
"woc_ui_action_exec_duration_seconds",
|
||
"UI action execution duration",
|
||
["priority"],
|
||
buckets=(0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0),
|
||
)
|
||
|
||
ui_action_timeout = Counter(
|
||
"woc_ui_action_timeout_total",
|
||
"UI action wait timeout count",
|
||
)
|
||
|
||
ui_action_rate_limited = Counter(
|
||
"woc_ui_action_rate_limited_total",
|
||
"UI action rate limited (queue full) count",
|
||
)
|
||
|
||
ui_action_fast_fail = Counter(
|
||
"woc_ui_action_fast_fail_total",
|
||
"UI action fast-fail count (wechat dead window)",
|
||
)
|
||
|
||
|
||
def set_circuit_state(name: str, state_int: int) -> None:
|
||
"""更新熔断器状态指标。
|
||
|
||
Args:
|
||
name: 熔断器名称
|
||
state_int: 0=closed, 1=open, 2=half_open
|
||
"""
|
||
circuit_state.labels(name=name).set(state_int)
|