refactor(ui-backend): 优化窗口校验与点击流程,提升缓存命中率与执行效率
1. 新增is_window_active接口并在xdotool后端实现,用于校验窗口焦点 2. 为xdotool后端添加主窗口ID缓存,减少重复xdotool调用 3. 精简click命令流程,移除--sync参数避免VNC环境死等 4. 简化post_verify执行逻辑,减少不必要的点击操作 5. 为缓存命中分支添加会话校验回退机制,处理窗口失焦/消失场景
This commit is contained in:
parent
ed2ee086de
commit
be93906635
@ -54,6 +54,16 @@ class BackendProtocol(ABC):
|
||||
async def activate_window(self, window_title: str) -> None:
|
||||
"""激活指定窗口(使其获得焦点)。"""
|
||||
|
||||
async def is_window_active(self, window_title: str) -> bool:
|
||||
"""校验指定标题的窗口是否是当前活动窗口(拥有焦点)。
|
||||
|
||||
默认实现返回 True(信任 activate_window 的结果)。
|
||||
XdotoolBackend 覆盖为真正调用 getactivewindow 对比。
|
||||
用于纯坐标模式下 _verify_session_still_open 校验窗口焦点,
|
||||
避免 cache_hit 分支命中已失焦的会话。
|
||||
"""
|
||||
return True
|
||||
|
||||
@property
|
||||
def capabilities(self) -> set[str]:
|
||||
"""返回后端支持的能力集合。子类可覆盖。"""
|
||||
|
||||
@ -53,6 +53,12 @@ class XdotoolBackend(BackendProtocol):
|
||||
)
|
||||
self._debug_queue: asyncio.Queue = asyncio.Queue()
|
||||
self._debug_task: Optional[asyncio.Task] = None
|
||||
# 主窗口 ID 缓存:window_title -> (window_id, expire_at)
|
||||
# 避免 cache_hit 路径上 _find_main_window_id 被重复调用 3-5 次
|
||||
# (activate_window + get_window_geometry + is_window_active 各调一次)
|
||||
# TTL 5s,窗口切换/关闭时由下次查询自然失效
|
||||
self._main_window_id_cache: dict[str, tuple[int, float]] = {}
|
||||
self._main_window_id_cache_ttl = 5.0
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 环境与子进程管理
|
||||
@ -134,9 +140,16 @@ class XdotoolBackend(BackendProtocol):
|
||||
# BackendProtocol 实现
|
||||
# ------------------------------------------------------------------
|
||||
async def click(self, x: int, y: int) -> None:
|
||||
"""合并 mousemove --sync + click 1 为单条 xdotool 命令。
|
||||
"""合并 mousemove + click 1 为单条 xdotool 命令。
|
||||
|
||||
关键:单条命令,避免分两次 fork 之间窗口焦点变化导致点击错位。
|
||||
|
||||
不使用 --sync:mousemove --sync 会阻塞等待 X server 确认鼠标 motion
|
||||
完成,在 VNC/Xvnc 环境下(pointer 被 grab、X server 不响应 motion
|
||||
事件、窗口未真正激活等)会无限期死等,触发 5s 命令超时。去掉 --sync
|
||||
与 activate_window 的处理保持一致(参见 L392 注释)。
|
||||
xdotool 内部串行执行 mousemove → click,X server 按序处理 motion 与
|
||||
button 事件,去掉 --sync 不影响点击精度。
|
||||
"""
|
||||
logger.info("[ui-backend] click start (%d,%d)", x, y)
|
||||
t0 = time.perf_counter()
|
||||
@ -145,7 +158,6 @@ class XdotoolBackend(BackendProtocol):
|
||||
[
|
||||
"xdotool",
|
||||
"mousemove",
|
||||
"--sync",
|
||||
str(x),
|
||||
str(y),
|
||||
"click",
|
||||
@ -259,6 +271,9 @@ class XdotoolBackend(BackendProtocol):
|
||||
3. 过滤掉面积 < min_area 的候选;
|
||||
4. 返回面积最大者。
|
||||
|
||||
带 5s TTL 缓存:cache_hit 路径上 activate_window + get_window_geometry
|
||||
+ is_window_active 会重复调用本方法,缓存避免 3-5 次 xdotool search。
|
||||
|
||||
Args:
|
||||
window_title: 窗口标题,如 "微信"
|
||||
min_area: 最小有效窗口面积(默认 10000 像素)
|
||||
@ -269,6 +284,14 @@ class XdotoolBackend(BackendProtocol):
|
||||
Raises:
|
||||
RuntimeError: 未找到有效窗口
|
||||
"""
|
||||
# 缓存命中检查
|
||||
cached = self._main_window_id_cache.get(window_title)
|
||||
if cached is not None:
|
||||
wid, expire_at = cached
|
||||
if time.monotonic() < expire_at:
|
||||
return wid
|
||||
self._main_window_id_cache.pop(window_title, None)
|
||||
|
||||
rc, stdout, _ = await self._run(
|
||||
["xdotool", "search", "--name", window_title]
|
||||
)
|
||||
@ -316,7 +339,12 @@ class XdotoolBackend(BackendProtocol):
|
||||
f"(candidates filtered by min_area={min_area})"
|
||||
)
|
||||
candidates.sort(key=lambda x: x[0], reverse=True)
|
||||
return candidates[0][1]
|
||||
main_id = candidates[0][1]
|
||||
# 写入缓存
|
||||
self._main_window_id_cache[window_title] = (
|
||||
main_id, time.monotonic() + self._main_window_id_cache_ttl
|
||||
)
|
||||
return main_id
|
||||
|
||||
async def get_window_geometry(self, window_title: str) -> WindowGeometry:
|
||||
"""按标题查窗口并返回几何。容忍多个匹配窗口,取面积最大主窗口。
|
||||
@ -429,6 +457,34 @@ class XdotoolBackend(BackendProtocol):
|
||||
window_id, window_title, (time.perf_counter() - t0) * 1000,
|
||||
)
|
||||
|
||||
async def is_window_active(self, window_title: str) -> bool:
|
||||
"""校验指定标题的窗口是否是当前活动窗口。
|
||||
|
||||
用于纯坐标模式下 _verify_session_still_open 校验窗口焦点:
|
||||
activate_window 调用后焦点校验可能超时(只记 warning 不 raise),
|
||||
此方法提供二次校验,若窗口已失焦则 cache 失效走完整流程。
|
||||
|
||||
Returns:
|
||||
True 表示窗口存在且是活动窗口;False 表示窗口不存在或非活动窗口
|
||||
"""
|
||||
try:
|
||||
rc, stdout, _ = await self._run(["xdotool", "getactivewindow"])
|
||||
if rc != 0:
|
||||
return False
|
||||
active_text = stdout.decode(errors="ignore").strip()
|
||||
try:
|
||||
active_id = int(active_text)
|
||||
except ValueError:
|
||||
return False
|
||||
main_id = await self._find_main_window_id(window_title)
|
||||
return active_id == main_id
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"[ui-backend] is_window_active exception title=%r: %s",
|
||||
window_title, exc,
|
||||
)
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 调试截图异步队列(生产环境关闭,调试模式后台写盘)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@ -107,6 +107,11 @@ class Flow:
|
||||
点击位置采用比例坐标,固定落在右侧聊天区域,避免命中左侧联系人列表、
|
||||
分割线或底部输入框导致的误操作。
|
||||
|
||||
异常路径下应尽量减少 xdotool 调用次数,避免恢复路径放大延迟。
|
||||
去掉 click 的 --sync 后不会卡死,但每条命令仍需 ~100ms,
|
||||
故精简为 4 步:Esc×3 → click 安全空白 → BackSpace×30 → Esc×2。
|
||||
post_verify 仅在 OpenCV 模式下有意义(纯坐标模式立即返回)。
|
||||
|
||||
Args:
|
||||
win_geom: 窗口几何(可选,用于点击空白区域与 post_verify)
|
||||
|
||||
@ -145,17 +150,8 @@ class Flow:
|
||||
await self.actions.backend.key_press("Escape", repeat=2)
|
||||
await asyncio.sleep(0.15)
|
||||
|
||||
# 5. 再次点击安全空白,确保焦点不在输入框
|
||||
if win_geom is not None and win_geom.width >= 200 and win_geom.height >= 200:
|
||||
blank_x = win_geom.x + int(win_geom.width * 0.75)
|
||||
blank_y = win_geom.y + int(win_geom.height * 0.40)
|
||||
left_panel_max_x = win_geom.x + int(win_geom.width * 0.42)
|
||||
if blank_x <= left_panel_max_x:
|
||||
blank_x = win_geom.x + int(win_geom.width * 0.75)
|
||||
await self.actions.backend.click(blank_x, blank_y)
|
||||
await asyncio.sleep(0.15)
|
||||
|
||||
# 6. post_verify:尝试用 main_view 模板校验
|
||||
# 5. post_verify:尝试用 main_view 模板校验
|
||||
# 纯坐标模式下 wait_for 立即返回(纯算术),不阻塞
|
||||
if win_geom is not None:
|
||||
try:
|
||||
elem = await self.actions.wait_for(
|
||||
|
||||
@ -83,14 +83,26 @@ class SendTextFlow(Flow):
|
||||
_WECHAT_WINDOW_TITLE
|
||||
)
|
||||
)
|
||||
await self._verify_session_still_open(ctx)
|
||||
ctx.current_state = FlowState.SESSION_OPENED
|
||||
logger.info(
|
||||
"[flow:send_text] cache_hit branch done geom=%dx%d (%.0fms)",
|
||||
ctx.win_geom.width, ctx.win_geom.height,
|
||||
(time.perf_counter() - t0) * 1000,
|
||||
)
|
||||
else:
|
||||
# 校验会话仍打开;失败时回退到完整流程,不直接返回失败
|
||||
try:
|
||||
await self._verify_session_still_open(ctx)
|
||||
ctx.current_state = FlowState.SESSION_OPENED
|
||||
logger.info(
|
||||
"[flow:send_text] cache_hit branch done geom=%dx%d (%.0fms)",
|
||||
ctx.win_geom.width, ctx.win_geom.height,
|
||||
(time.perf_counter() - t0) * 1000,
|
||||
)
|
||||
except PermanentError as verify_exc:
|
||||
logger.info(
|
||||
"[flow:send_text] cache_hit verify failed, fallback to full flow: %s (%.0fms)",
|
||||
verify_exc, (time.perf_counter() - t0) * 1000,
|
||||
)
|
||||
# 失效缓存,走完整流程
|
||||
if self.session_cache is not None:
|
||||
self.session_cache.invalidate(ctx.to_wxid)
|
||||
cache_hit = False
|
||||
|
||||
if not cache_hit:
|
||||
# 完整流程
|
||||
await self._activate(ctx)
|
||||
await self._click_search_box(ctx)
|
||||
@ -377,12 +389,58 @@ class SendTextFlow(Flow):
|
||||
return False
|
||||
|
||||
async def _verify_session_still_open(self, ctx: FlowContext) -> None:
|
||||
"""会话缓存命中时校验会话仍打开(input_box 存在)。"""
|
||||
"""会话缓存命中时校验会话仍打开。
|
||||
|
||||
纯坐标模式(WOC_DISABLE_OPENCV=1)下 wait_for("input_box") 纯算术
|
||||
返回,无法真正校验 UI 状态。此时改用 is_window_active 校验窗口焦点:
|
||||
若微信窗口已失焦(用户切到其他应用),cache 失效走完整流程。
|
||||
会话是否真的打开仍靠后续 SSE verify 兜底(content 不匹配则 invalidate)。
|
||||
|
||||
OpenCV 模式下保留原逻辑(wait_for input_box 模板匹配)。
|
||||
|
||||
异常处理:get_window_geometry 的 RuntimeError(窗口消失)也转为
|
||||
PermanentError,触发 cache_hit 回退到完整流程,避免冒泡为
|
||||
BRIDGE_INTERNAL_ERROR。
|
||||
"""
|
||||
t0 = time.perf_counter()
|
||||
# 获取窗口几何(缓存命中时未走 _activate),并校验/窄化类型
|
||||
ctx.win_geom = self._ensure_valid_geometry(
|
||||
await self.actions.backend.get_window_geometry(_WECHAT_WINDOW_TITLE)
|
||||
)
|
||||
try:
|
||||
ctx.win_geom = self._ensure_valid_geometry(
|
||||
await self.actions.backend.get_window_geometry(_WECHAT_WINDOW_TITLE)
|
||||
)
|
||||
except (RuntimeError, PermanentError) as exc:
|
||||
logger.warning(
|
||||
"[flow:send_text] session_still_open FAIL: get_window_geometry failed: %s (%.0fms)",
|
||||
exc, (time.perf_counter() - t0) * 1000,
|
||||
)
|
||||
raise PermanentError(
|
||||
"WINDOW_NOT_FOUND",
|
||||
f"session cache verify failed: window not found: {exc}",
|
||||
)
|
||||
from woc_bridge.ui.actions import _DISABLE_OPENCV
|
||||
|
||||
if _DISABLE_OPENCV:
|
||||
# 纯坐标模式:校验窗口焦点,避免命中已失焦的会话
|
||||
is_active = await self.actions.backend.is_window_active(
|
||||
_WECHAT_WINDOW_TITLE
|
||||
)
|
||||
elapsed_ms = (time.perf_counter() - t0) * 1000
|
||||
if not is_active:
|
||||
logger.warning(
|
||||
"[flow:send_text] session_still_open FAIL: window not active (%.0fms)",
|
||||
elapsed_ms,
|
||||
)
|
||||
raise PermanentError(
|
||||
"ELEMENT_NOT_FOUND",
|
||||
"session cache verify failed: window not active",
|
||||
)
|
||||
logger.info(
|
||||
"[flow:send_text] session_still_open OK (window active, %.0fms)",
|
||||
elapsed_ms,
|
||||
)
|
||||
return
|
||||
|
||||
# OpenCV 模式:用 input_box 模板匹配校验
|
||||
try:
|
||||
elem = await self.actions.wait_for(
|
||||
"input_box", ctx.win_geom, timeout=2.0, interval=0.3
|
||||
@ -400,6 +458,8 @@ class SendTextFlow(Flow):
|
||||
"[flow:send_text] session_still_open OK (%.0fms)",
|
||||
(time.perf_counter() - t0) * 1000,
|
||||
)
|
||||
except PermanentError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"[flow:send_text] session_still_open exception: %s (%.0fms)",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user