Kris
054c4676aa
主要变更： 1. 新增SSEVerifyBus实现基于推送的发送结果校验，替代原DB轮询方案 2. 移除FlowOrchestrator中的DB熔断器，避免雪崩风险 3. 重构SendTextFlow，改用SSE校验流程，删除原DB校验相关代码 4. 新增RoiSpec与ROI搜索支持，优化UI元素匹配精度 5. 更新所有分辨率配置文件，添加ROI配置与调整搜索框坐标 6. 新增WOC_DISABLE_OPENCV环境变量开关，支持纯几何定位模式 7. 清理FlowContext中废弃的before_msg_id字段
370 lines
13 KiB
Python
370 lines
13 KiB
Python
"""L3 Actions 原子动作库：find / click / type / wait / screenshot。
|
||
|
||
Actions 是 L1 Backend 与 L2 Locator 的组合层：给定 kind 与窗口几何，
|
||
Locator 解析为 ElementHandle（图像优先、几何兜底），Backend 据此执行点击/输入。
|
||
L4 Flow 通过 Actions 组合出完整业务流程（如 SendTextFlow）。
|
||
|
||
策略：
|
||
- 图像路径（P2 启用）：sel.by_image 非空且 locator.is_circuited 为 False 且 opencv 非 None 时优先
|
||
- 命中：record_image_success，返回 strategy=image
|
||
- 未命中：record_image_failure；require_image=True 抛 ElementNotFoundError；否则回退 _geom_find
|
||
- 几何兜底（P1 默认）：按 GeomSpec 计算绝对坐标
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import logging
|
||
import os
|
||
import time
|
||
from typing import Optional
|
||
|
||
from woc_bridge.ui.backends.base import BackendProtocol, WindowGeometry
|
||
from woc_bridge.ui.locators.base import ElementHandle, Selector
|
||
from woc_bridge.ui.locators.registry import LocatorRegistry
|
||
|
||
logger = logging.getLogger("woc-bridge")
|
||
|
||
# 纯坐标模式开关：WOC_DISABLE_OPENCV=1 时跳过 OpenCV 模板匹配，
|
||
# find_element 直接走几何兜底（by_geom 比例坐标），不截图不匹配。
|
||
# 适用于模板图像不可靠的场景，靠窗口几何比例定位元素。
|
||
# 启动时读取一次，运行时不再重复读环境变量。
|
||
_DISABLE_OPENCV = os.environ.get("WOC_DISABLE_OPENCV", "").strip() in (
|
||
"1", "true", "True", "TRUE", "yes",
|
||
)
|
||
|
||
|
||
class ElementNotFoundError(Exception):
|
||
"""元素未找到（图像匹配失败且 require_image=True，或几何规格缺失）。
|
||
|
||
TODO: P3 时统一到 errors.py 的 FlowError(code="ELEMENT_NOT_FOUND")。
|
||
"""
|
||
|
||
pass
|
||
|
||
|
||
class Actions:
|
||
"""L3 原子动作库：组合 Backend（L1）与 Locator（L2）。
|
||
|
||
所有方法均为 async，由 L4 Flow 编排调用。
|
||
"""
|
||
|
||
def __init__(
|
||
self, backend: BackendProtocol, locator: LocatorRegistry
|
||
) -> None:
|
||
"""初始化 Actions。
|
||
|
||
Args:
|
||
backend: L1 Backend 实例（如 XdotoolBackend）
|
||
locator: L2 LocatorRegistry 实例
|
||
"""
|
||
self.backend = backend
|
||
self.locator = locator
|
||
# 截图缓存：同一次 flow 内连续查找可避免重复截图（CPU/IO 优化）
|
||
self._shot_cache: Optional[tuple[float, bytes]] = None
|
||
self._shot_cache_ttl_ms = 150
|
||
|
||
def _get_cached_screenshot(self) -> Optional[bytes]:
|
||
"""返回未过期的缓存截图，若无则 None。"""
|
||
if self._shot_cache is None:
|
||
return None
|
||
ts, shot = self._shot_cache
|
||
if (time.monotonic() - ts) * 1000 > self._shot_cache_ttl_ms:
|
||
self._shot_cache = None
|
||
return None
|
||
return shot
|
||
|
||
def _set_cached_screenshot(self, shot: bytes) -> None:
|
||
"""写入截图缓存。"""
|
||
self._shot_cache = (time.monotonic(), shot)
|
||
|
||
# ------------------------------------------------------------------
|
||
# 元素查找
|
||
# ------------------------------------------------------------------
|
||
async def find_element(
|
||
self, kind: str, win_geom: WindowGeometry
|
||
) -> ElementHandle:
|
||
"""查找元素，优先图像匹配，失败回退几何兜底。
|
||
|
||
Args:
|
||
kind: 元素类别键（如 search_box / send_button）
|
||
win_geom: 当前窗口几何（绝对坐标 + 宽高）
|
||
|
||
Returns:
|
||
ElementHandle：命中元素（含绝对坐标与策略）
|
||
|
||
Raises:
|
||
ElementNotFoundError: require_image=True 且图像匹配失败，
|
||
或 by_geom 为空且无图像命中
|
||
"""
|
||
sel = self.locator.get(kind)
|
||
|
||
# 纯坐标模式：WOC_DISABLE_OPENCV=1 时直接走几何兜底，
|
||
# 跳过截图与 OpenCV 模板匹配，按 Selector.by_geom 比例坐标定位。
|
||
# 返回 strategy="geom"，confidence=1.0。
|
||
if _DISABLE_OPENCV:
|
||
logger.debug(
|
||
"[action] opencv disabled (WOC_DISABLE_OPENCV=1), "
|
||
"use geom directly kind=%s",
|
||
kind,
|
||
)
|
||
return self._geom_find(sel, win_geom)
|
||
|
||
# 图像路径（P2 启用，P1 阶段 opencv=None 直接跳过）
|
||
if (
|
||
sel.by_image
|
||
and not self.locator.is_circuited(kind)
|
||
and self.locator.opencv is not None
|
||
):
|
||
shot = self._get_cached_screenshot()
|
||
if shot is None:
|
||
shot = await self.backend.screenshot()
|
||
self._set_cached_screenshot(shot)
|
||
# 计算 OpenCV 搜索 ROI：优先用 Selector.roi（相对窗口的比例矩形），
|
||
# 无则回退到全窗口。ROI 限制可防止模板误匹配到窗口其他相似元素
|
||
# （如 input_box 模板错误时匹配到窗口中部，参见 false-positive 事件）。
|
||
roi = self._compute_roi(sel, win_geom)
|
||
try:
|
||
result = await self.locator.opencv.find_template(
|
||
shot,
|
||
sel.by_image,
|
||
threshold=sel.threshold,
|
||
roi=roi,
|
||
)
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"[action] opencv.find_template exception kind=%s: %s",
|
||
kind,
|
||
exc,
|
||
)
|
||
result = None
|
||
|
||
if result is not None:
|
||
# result = (x, y, w, h, confidence)（绝对坐标，含 ROI 偏移）
|
||
x, y, w, h, conf = result
|
||
self.locator.record_image_success(kind)
|
||
logger.info(
|
||
"[action] image match success kind=%s at (%d,%d) conf=%.3f",
|
||
kind, x + w // 2, y + h // 2, conf,
|
||
)
|
||
return ElementHandle(
|
||
selector=sel,
|
||
x=x + w // 2,
|
||
y=y + h // 2,
|
||
w=w,
|
||
h=h,
|
||
confidence=conf,
|
||
strategy="image",
|
||
)
|
||
|
||
# 图像未命中
|
||
self.locator.record_image_failure(kind)
|
||
logger.warning(
|
||
"[action] image match failed kind=%s, fall back to geom", kind
|
||
)
|
||
if sel.require_image:
|
||
raise ElementNotFoundError(
|
||
f"image_match_failed kind={kind}"
|
||
)
|
||
|
||
logger.debug("[action] geom fallback kind=%s", kind)
|
||
return self._geom_find(sel, win_geom)
|
||
|
||
async def click_element(
|
||
self, kind: str, win_geom: WindowGeometry
|
||
) -> ElementHandle:
|
||
"""查找并点击元素，返回 ElementHandle。
|
||
|
||
Args:
|
||
kind: 元素类别键
|
||
win_geom: 当前窗口几何
|
||
|
||
Returns:
|
||
命中的 ElementHandle
|
||
"""
|
||
elem = await self.find_element(kind, win_geom)
|
||
await self.backend.click(elem.x, elem.y)
|
||
logger.info(
|
||
"[action] click %s at (%d,%d) strategy=%s conf=%.3f",
|
||
kind,
|
||
elem.x,
|
||
elem.y,
|
||
elem.strategy,
|
||
elem.confidence,
|
||
)
|
||
return elem
|
||
|
||
async def type_into(
|
||
self,
|
||
kind: str,
|
||
text: str,
|
||
win_geom: WindowGeometry,
|
||
clear_first: bool = True,
|
||
) -> None:
|
||
"""点击元素聚焦后输入文本，可选先清空。
|
||
|
||
Args:
|
||
kind: 元素类别键（通常为 input_box）
|
||
text: 要输入的文本
|
||
win_geom: 当前窗口几何
|
||
clear_first: 是否先 BackSpace×30 清空旧内容
|
||
"""
|
||
t0 = time.perf_counter()
|
||
logger.info(
|
||
"[action] type_into start kind=%s text_len=%d clear_first=%s",
|
||
kind, len(text), clear_first,
|
||
)
|
||
await self.click_element(kind, win_geom)
|
||
if clear_first:
|
||
await self.backend.key_press("BackSpace", repeat=30)
|
||
await self.backend.type_text(text)
|
||
logger.info(
|
||
"[action] type_into done kind=%s text_len=%d (%.0fms)",
|
||
kind, len(text), (time.perf_counter() - t0) * 1000,
|
||
)
|
||
|
||
async def wait_for(
|
||
self,
|
||
kind: str,
|
||
win_geom: WindowGeometry,
|
||
timeout: float = 5.0,
|
||
interval: float = 0.3,
|
||
) -> Optional[ElementHandle]:
|
||
"""轮询等待元素出现，超时返回 None。
|
||
|
||
Args:
|
||
kind: 元素类别键
|
||
win_geom: 当前窗口几何
|
||
timeout: 最大等待秒数
|
||
interval: 轮询间隔秒数
|
||
|
||
Returns:
|
||
ElementHandle 或 None（超时）
|
||
"""
|
||
t0 = time.perf_counter()
|
||
deadline = time.monotonic() + timeout
|
||
attempts = 0
|
||
last_err: Optional[Exception] = None
|
||
while time.monotonic() < deadline:
|
||
attempts += 1
|
||
try:
|
||
elem = await self.find_element(kind, win_geom)
|
||
if elem:
|
||
logger.info(
|
||
"[action] wait_for OK kind=%s strategy=%s attempts=%d (%.0fms)",
|
||
kind, elem.strategy, attempts,
|
||
(time.perf_counter() - t0) * 1000,
|
||
)
|
||
return elem
|
||
except ElementNotFoundError as e:
|
||
last_err = e
|
||
await asyncio.sleep(interval)
|
||
logger.warning(
|
||
"[action] wait_for TIMEOUT kind=%s attempts=%d timeout=%.1fs last_err=%s",
|
||
kind, attempts, timeout, last_err,
|
||
)
|
||
return None
|
||
|
||
async def screenshot(self) -> bytes:
|
||
"""截图代理方法，带 150ms 缓存。"""
|
||
logger.debug("[action] screenshot start")
|
||
data = self._get_cached_screenshot()
|
||
if data is None:
|
||
data = await self.backend.screenshot()
|
||
self._set_cached_screenshot(data)
|
||
logger.debug(
|
||
"[action] screenshot done bytes=%d", len(data) if data else 0
|
||
)
|
||
return data
|
||
|
||
# ------------------------------------------------------------------
|
||
# 几何兜底
|
||
# ------------------------------------------------------------------
|
||
def _compute_roi(
|
||
self, selector: Selector, win_geom: WindowGeometry
|
||
) -> tuple[int, int, int, int]:
|
||
"""计算 OpenCV 搜索 ROI（绝对像素坐标）。
|
||
|
||
优先使用 Selector.roi（相对窗口的比例矩形），无则回退到全窗口。
|
||
输出格式为 (x, y, w, h)，与 OpenCVBackend.find_template 的 roi 参数一致。
|
||
边界由 opencv.py 内部裁剪保护，这里只做基础 clamp 防负值。
|
||
|
||
Args:
|
||
selector: 元素选择器（可能含 roi）
|
||
win_geom: 当前窗口几何
|
||
|
||
Returns:
|
||
(x, y, w, h) 绝对像素 ROI
|
||
"""
|
||
if selector.roi is None:
|
||
return (win_geom.x, win_geom.y, win_geom.width, win_geom.height)
|
||
r = selector.roi
|
||
abs_x = win_geom.x + int(win_geom.width * r.x_ratio)
|
||
abs_y = win_geom.y + int(win_geom.height * r.y_ratio)
|
||
abs_w = int(win_geom.width * r.w_ratio)
|
||
abs_h = int(win_geom.height * r.h_ratio)
|
||
# 基础保护：防止负值或零尺寸
|
||
abs_w = max(abs_w, 1)
|
||
abs_h = max(abs_h, 1)
|
||
logger.debug(
|
||
"[action] compute_roi kind=%s win=(%d,%d,%dx%d) "
|
||
"roi_ratio=(%.3f,%.3f,%.3f,%.3f) → abs=(%d,%d,%dx%d)",
|
||
selector.kind,
|
||
win_geom.x, win_geom.y, win_geom.width, win_geom.height,
|
||
r.x_ratio, r.y_ratio, r.w_ratio, r.h_ratio,
|
||
abs_x, abs_y, abs_w, abs_h,
|
||
)
|
||
return (abs_x, abs_y, abs_w, abs_h)
|
||
|
||
def _geom_find(
|
||
self, selector: Selector, win_geom: WindowGeometry
|
||
) -> ElementHandle:
|
||
"""按 GeomSpec 计算绝对坐标，返回 ElementHandle。
|
||
|
||
支持两种 relative_to 语义：
|
||
- window: 以窗口左上角为原点
|
||
base_x = win_geom.x + win_geom.width * g.x_ratio + g.x_offset
|
||
base_y = win_geom.y + win_geom.height * g.y_ratio + g.y_offset
|
||
- window_bottom_right: 以窗口右下角为原点（offset 含正负号）
|
||
base_x = win_geom.x + win_geom.width + g.x_offset
|
||
base_y = win_geom.y + win_geom.height + g.y_offset
|
||
|
||
Args:
|
||
selector: 元素选择器（必须含 by_geom）
|
||
win_geom: 当前窗口几何
|
||
|
||
Returns:
|
||
ElementHandle（strategy=geom, confidence=1.0）
|
||
|
||
Raises:
|
||
ElementNotFoundError: selector.by_geom 为空
|
||
"""
|
||
if selector.by_geom is None:
|
||
raise ElementNotFoundError(
|
||
f"no geom spec kind={selector.kind}"
|
||
)
|
||
g = selector.by_geom
|
||
if g.relative_to == "window_bottom_right":
|
||
base_x = win_geom.x + win_geom.width + g.x_offset
|
||
base_y = win_geom.y + win_geom.height + g.y_offset
|
||
else:
|
||
base_x = win_geom.x + win_geom.width * g.x_ratio + g.x_offset
|
||
base_y = win_geom.y + win_geom.height * g.y_ratio + g.y_offset
|
||
logger.debug(
|
||
"[action] geom_find kind=%s relative_to=%s win=(%d,%d,%dx%d) "
|
||
"ratio=(%.3f,%.3f) offset=(%d,%d) → abs=(%d,%d)",
|
||
selector.kind, g.relative_to,
|
||
win_geom.x, win_geom.y, win_geom.width, win_geom.height,
|
||
g.x_ratio, g.y_ratio, g.x_offset, g.y_offset,
|
||
int(base_x), int(base_y),
|
||
)
|
||
return ElementHandle(
|
||
selector=selector,
|
||
x=int(base_x),
|
||
y=int(base_y),
|
||
w=0,
|
||
h=0,
|
||
confidence=1.0,
|
||
strategy="geom",
|
||
)
|