refactor(ui-backend): 优化窗口校验与点击流程,提升缓存命中率与执行效率
1. 新增is_window_active接口并在xdotool后端实现,用于校验窗口焦点 2. 为xdotool后端添加主窗口ID缓存,减少重复xdotool调用 3. 精简click命令流程,移除--sync参数避免VNC环境死等 4. 简化post_verify执行逻辑,减少不必要的点击操作 5. 为缓存命中分支添加会话校验回退机制,处理窗口失焦/消失场景
This commit is contained in:
parent
ed2ee086de
commit
be93906635
@ -54,6 +54,16 @@ class BackendProtocol(ABC):
|
|||||||
async def activate_window(self, window_title: str) -> None:
|
async def activate_window(self, window_title: str) -> None:
|
||||||
"""激活指定窗口(使其获得焦点)。"""
|
"""激活指定窗口(使其获得焦点)。"""
|
||||||
|
|
||||||
|
async def is_window_active(self, window_title: str) -> bool:
|
||||||
|
"""校验指定标题的窗口是否是当前活动窗口(拥有焦点)。
|
||||||
|
|
||||||
|
默认实现返回 True(信任 activate_window 的结果)。
|
||||||
|
XdotoolBackend 覆盖为真正调用 getactivewindow 对比。
|
||||||
|
用于纯坐标模式下 _verify_session_still_open 校验窗口焦点,
|
||||||
|
避免 cache_hit 分支命中已失焦的会话。
|
||||||
|
"""
|
||||||
|
return True
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def capabilities(self) -> set[str]:
|
def capabilities(self) -> set[str]:
|
||||||
"""返回后端支持的能力集合。子类可覆盖。"""
|
"""返回后端支持的能力集合。子类可覆盖。"""
|
||||||
|
|||||||
@ -53,6 +53,12 @@ class XdotoolBackend(BackendProtocol):
|
|||||||
)
|
)
|
||||||
self._debug_queue: asyncio.Queue = asyncio.Queue()
|
self._debug_queue: asyncio.Queue = asyncio.Queue()
|
||||||
self._debug_task: Optional[asyncio.Task] = None
|
self._debug_task: Optional[asyncio.Task] = None
|
||||||
|
# 主窗口 ID 缓存:window_title -> (window_id, expire_at)
|
||||||
|
# 避免 cache_hit 路径上 _find_main_window_id 被重复调用 3-5 次
|
||||||
|
# (activate_window + get_window_geometry + is_window_active 各调一次)
|
||||||
|
# TTL 5s,窗口切换/关闭时由下次查询自然失效
|
||||||
|
self._main_window_id_cache: dict[str, tuple[int, float]] = {}
|
||||||
|
self._main_window_id_cache_ttl = 5.0
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# 环境与子进程管理
|
# 环境与子进程管理
|
||||||
@ -134,9 +140,16 @@ class XdotoolBackend(BackendProtocol):
|
|||||||
# BackendProtocol 实现
|
# BackendProtocol 实现
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
async def click(self, x: int, y: int) -> None:
|
async def click(self, x: int, y: int) -> None:
|
||||||
"""合并 mousemove --sync + click 1 为单条 xdotool 命令。
|
"""合并 mousemove + click 1 为单条 xdotool 命令。
|
||||||
|
|
||||||
关键:单条命令,避免分两次 fork 之间窗口焦点变化导致点击错位。
|
关键:单条命令,避免分两次 fork 之间窗口焦点变化导致点击错位。
|
||||||
|
|
||||||
|
不使用 --sync:mousemove --sync 会阻塞等待 X server 确认鼠标 motion
|
||||||
|
完成,在 VNC/Xvnc 环境下(pointer 被 grab、X server 不响应 motion
|
||||||
|
事件、窗口未真正激活等)会无限期死等,触发 5s 命令超时。去掉 --sync
|
||||||
|
与 activate_window 的处理保持一致(参见 L392 注释)。
|
||||||
|
xdotool 内部串行执行 mousemove → click,X server 按序处理 motion 与
|
||||||
|
button 事件,去掉 --sync 不影响点击精度。
|
||||||
"""
|
"""
|
||||||
logger.info("[ui-backend] click start (%d,%d)", x, y)
|
logger.info("[ui-backend] click start (%d,%d)", x, y)
|
||||||
t0 = time.perf_counter()
|
t0 = time.perf_counter()
|
||||||
@ -145,7 +158,6 @@ class XdotoolBackend(BackendProtocol):
|
|||||||
[
|
[
|
||||||
"xdotool",
|
"xdotool",
|
||||||
"mousemove",
|
"mousemove",
|
||||||
"--sync",
|
|
||||||
str(x),
|
str(x),
|
||||||
str(y),
|
str(y),
|
||||||
"click",
|
"click",
|
||||||
@ -259,6 +271,9 @@ class XdotoolBackend(BackendProtocol):
|
|||||||
3. 过滤掉面积 < min_area 的候选;
|
3. 过滤掉面积 < min_area 的候选;
|
||||||
4. 返回面积最大者。
|
4. 返回面积最大者。
|
||||||
|
|
||||||
|
带 5s TTL 缓存:cache_hit 路径上 activate_window + get_window_geometry
|
||||||
|
+ is_window_active 会重复调用本方法,缓存避免 3-5 次 xdotool search。
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
window_title: 窗口标题,如 "微信"
|
window_title: 窗口标题,如 "微信"
|
||||||
min_area: 最小有效窗口面积(默认 10000 像素)
|
min_area: 最小有效窗口面积(默认 10000 像素)
|
||||||
@ -269,6 +284,14 @@ class XdotoolBackend(BackendProtocol):
|
|||||||
Raises:
|
Raises:
|
||||||
RuntimeError: 未找到有效窗口
|
RuntimeError: 未找到有效窗口
|
||||||
"""
|
"""
|
||||||
|
# 缓存命中检查
|
||||||
|
cached = self._main_window_id_cache.get(window_title)
|
||||||
|
if cached is not None:
|
||||||
|
wid, expire_at = cached
|
||||||
|
if time.monotonic() < expire_at:
|
||||||
|
return wid
|
||||||
|
self._main_window_id_cache.pop(window_title, None)
|
||||||
|
|
||||||
rc, stdout, _ = await self._run(
|
rc, stdout, _ = await self._run(
|
||||||
["xdotool", "search", "--name", window_title]
|
["xdotool", "search", "--name", window_title]
|
||||||
)
|
)
|
||||||
@ -316,7 +339,12 @@ class XdotoolBackend(BackendProtocol):
|
|||||||
f"(candidates filtered by min_area={min_area})"
|
f"(candidates filtered by min_area={min_area})"
|
||||||
)
|
)
|
||||||
candidates.sort(key=lambda x: x[0], reverse=True)
|
candidates.sort(key=lambda x: x[0], reverse=True)
|
||||||
return candidates[0][1]
|
main_id = candidates[0][1]
|
||||||
|
# 写入缓存
|
||||||
|
self._main_window_id_cache[window_title] = (
|
||||||
|
main_id, time.monotonic() + self._main_window_id_cache_ttl
|
||||||
|
)
|
||||||
|
return main_id
|
||||||
|
|
||||||
async def get_window_geometry(self, window_title: str) -> WindowGeometry:
|
async def get_window_geometry(self, window_title: str) -> WindowGeometry:
|
||||||
"""按标题查窗口并返回几何。容忍多个匹配窗口,取面积最大主窗口。
|
"""按标题查窗口并返回几何。容忍多个匹配窗口,取面积最大主窗口。
|
||||||
@ -429,6 +457,34 @@ class XdotoolBackend(BackendProtocol):
|
|||||||
window_id, window_title, (time.perf_counter() - t0) * 1000,
|
window_id, window_title, (time.perf_counter() - t0) * 1000,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
async def is_window_active(self, window_title: str) -> bool:
|
||||||
|
"""校验指定标题的窗口是否是当前活动窗口。
|
||||||
|
|
||||||
|
用于纯坐标模式下 _verify_session_still_open 校验窗口焦点:
|
||||||
|
activate_window 调用后焦点校验可能超时(只记 warning 不 raise),
|
||||||
|
此方法提供二次校验,若窗口已失焦则 cache 失效走完整流程。
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True 表示窗口存在且是活动窗口;False 表示窗口不存在或非活动窗口
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
rc, stdout, _ = await self._run(["xdotool", "getactivewindow"])
|
||||||
|
if rc != 0:
|
||||||
|
return False
|
||||||
|
active_text = stdout.decode(errors="ignore").strip()
|
||||||
|
try:
|
||||||
|
active_id = int(active_text)
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
main_id = await self._find_main_window_id(window_title)
|
||||||
|
return active_id == main_id
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(
|
||||||
|
"[ui-backend] is_window_active exception title=%r: %s",
|
||||||
|
window_title, exc,
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# 调试截图异步队列(生产环境关闭,调试模式后台写盘)
|
# 调试截图异步队列(生产环境关闭,调试模式后台写盘)
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|||||||
@ -107,6 +107,11 @@ class Flow:
|
|||||||
点击位置采用比例坐标,固定落在右侧聊天区域,避免命中左侧联系人列表、
|
点击位置采用比例坐标,固定落在右侧聊天区域,避免命中左侧联系人列表、
|
||||||
分割线或底部输入框导致的误操作。
|
分割线或底部输入框导致的误操作。
|
||||||
|
|
||||||
|
异常路径下应尽量减少 xdotool 调用次数,避免恢复路径放大延迟。
|
||||||
|
去掉 click 的 --sync 后不会卡死,但每条命令仍需 ~100ms,
|
||||||
|
故精简为 4 步:Esc×3 → click 安全空白 → BackSpace×30 → Esc×2。
|
||||||
|
post_verify 仅在 OpenCV 模式下有意义(纯坐标模式立即返回)。
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
win_geom: 窗口几何(可选,用于点击空白区域与 post_verify)
|
win_geom: 窗口几何(可选,用于点击空白区域与 post_verify)
|
||||||
|
|
||||||
@ -145,17 +150,8 @@ class Flow:
|
|||||||
await self.actions.backend.key_press("Escape", repeat=2)
|
await self.actions.backend.key_press("Escape", repeat=2)
|
||||||
await asyncio.sleep(0.15)
|
await asyncio.sleep(0.15)
|
||||||
|
|
||||||
# 5. 再次点击安全空白,确保焦点不在输入框
|
# 5. post_verify:尝试用 main_view 模板校验
|
||||||
if win_geom is not None and win_geom.width >= 200 and win_geom.height >= 200:
|
# 纯坐标模式下 wait_for 立即返回(纯算术),不阻塞
|
||||||
blank_x = win_geom.x + int(win_geom.width * 0.75)
|
|
||||||
blank_y = win_geom.y + int(win_geom.height * 0.40)
|
|
||||||
left_panel_max_x = win_geom.x + int(win_geom.width * 0.42)
|
|
||||||
if blank_x <= left_panel_max_x:
|
|
||||||
blank_x = win_geom.x + int(win_geom.width * 0.75)
|
|
||||||
await self.actions.backend.click(blank_x, blank_y)
|
|
||||||
await asyncio.sleep(0.15)
|
|
||||||
|
|
||||||
# 6. post_verify:尝试用 main_view 模板校验
|
|
||||||
if win_geom is not None:
|
if win_geom is not None:
|
||||||
try:
|
try:
|
||||||
elem = await self.actions.wait_for(
|
elem = await self.actions.wait_for(
|
||||||
|
|||||||
@ -83,14 +83,26 @@ class SendTextFlow(Flow):
|
|||||||
_WECHAT_WINDOW_TITLE
|
_WECHAT_WINDOW_TITLE
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
await self._verify_session_still_open(ctx)
|
# 校验会话仍打开;失败时回退到完整流程,不直接返回失败
|
||||||
ctx.current_state = FlowState.SESSION_OPENED
|
try:
|
||||||
logger.info(
|
await self._verify_session_still_open(ctx)
|
||||||
"[flow:send_text] cache_hit branch done geom=%dx%d (%.0fms)",
|
ctx.current_state = FlowState.SESSION_OPENED
|
||||||
ctx.win_geom.width, ctx.win_geom.height,
|
logger.info(
|
||||||
(time.perf_counter() - t0) * 1000,
|
"[flow:send_text] cache_hit branch done geom=%dx%d (%.0fms)",
|
||||||
)
|
ctx.win_geom.width, ctx.win_geom.height,
|
||||||
else:
|
(time.perf_counter() - t0) * 1000,
|
||||||
|
)
|
||||||
|
except PermanentError as verify_exc:
|
||||||
|
logger.info(
|
||||||
|
"[flow:send_text] cache_hit verify failed, fallback to full flow: %s (%.0fms)",
|
||||||
|
verify_exc, (time.perf_counter() - t0) * 1000,
|
||||||
|
)
|
||||||
|
# 失效缓存,走完整流程
|
||||||
|
if self.session_cache is not None:
|
||||||
|
self.session_cache.invalidate(ctx.to_wxid)
|
||||||
|
cache_hit = False
|
||||||
|
|
||||||
|
if not cache_hit:
|
||||||
# 完整流程
|
# 完整流程
|
||||||
await self._activate(ctx)
|
await self._activate(ctx)
|
||||||
await self._click_search_box(ctx)
|
await self._click_search_box(ctx)
|
||||||
@ -377,12 +389,58 @@ class SendTextFlow(Flow):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
async def _verify_session_still_open(self, ctx: FlowContext) -> None:
|
async def _verify_session_still_open(self, ctx: FlowContext) -> None:
|
||||||
"""会话缓存命中时校验会话仍打开(input_box 存在)。"""
|
"""会话缓存命中时校验会话仍打开。
|
||||||
|
|
||||||
|
纯坐标模式(WOC_DISABLE_OPENCV=1)下 wait_for("input_box") 纯算术
|
||||||
|
返回,无法真正校验 UI 状态。此时改用 is_window_active 校验窗口焦点:
|
||||||
|
若微信窗口已失焦(用户切到其他应用),cache 失效走完整流程。
|
||||||
|
会话是否真的打开仍靠后续 SSE verify 兜底(content 不匹配则 invalidate)。
|
||||||
|
|
||||||
|
OpenCV 模式下保留原逻辑(wait_for input_box 模板匹配)。
|
||||||
|
|
||||||
|
异常处理:get_window_geometry 的 RuntimeError(窗口消失)也转为
|
||||||
|
PermanentError,触发 cache_hit 回退到完整流程,避免冒泡为
|
||||||
|
BRIDGE_INTERNAL_ERROR。
|
||||||
|
"""
|
||||||
t0 = time.perf_counter()
|
t0 = time.perf_counter()
|
||||||
# 获取窗口几何(缓存命中时未走 _activate),并校验/窄化类型
|
# 获取窗口几何(缓存命中时未走 _activate),并校验/窄化类型
|
||||||
ctx.win_geom = self._ensure_valid_geometry(
|
try:
|
||||||
await self.actions.backend.get_window_geometry(_WECHAT_WINDOW_TITLE)
|
ctx.win_geom = self._ensure_valid_geometry(
|
||||||
)
|
await self.actions.backend.get_window_geometry(_WECHAT_WINDOW_TITLE)
|
||||||
|
)
|
||||||
|
except (RuntimeError, PermanentError) as exc:
|
||||||
|
logger.warning(
|
||||||
|
"[flow:send_text] session_still_open FAIL: get_window_geometry failed: %s (%.0fms)",
|
||||||
|
exc, (time.perf_counter() - t0) * 1000,
|
||||||
|
)
|
||||||
|
raise PermanentError(
|
||||||
|
"WINDOW_NOT_FOUND",
|
||||||
|
f"session cache verify failed: window not found: {exc}",
|
||||||
|
)
|
||||||
|
from woc_bridge.ui.actions import _DISABLE_OPENCV
|
||||||
|
|
||||||
|
if _DISABLE_OPENCV:
|
||||||
|
# 纯坐标模式:校验窗口焦点,避免命中已失焦的会话
|
||||||
|
is_active = await self.actions.backend.is_window_active(
|
||||||
|
_WECHAT_WINDOW_TITLE
|
||||||
|
)
|
||||||
|
elapsed_ms = (time.perf_counter() - t0) * 1000
|
||||||
|
if not is_active:
|
||||||
|
logger.warning(
|
||||||
|
"[flow:send_text] session_still_open FAIL: window not active (%.0fms)",
|
||||||
|
elapsed_ms,
|
||||||
|
)
|
||||||
|
raise PermanentError(
|
||||||
|
"ELEMENT_NOT_FOUND",
|
||||||
|
"session cache verify failed: window not active",
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"[flow:send_text] session_still_open OK (window active, %.0fms)",
|
||||||
|
elapsed_ms,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
# OpenCV 模式:用 input_box 模板匹配校验
|
||||||
try:
|
try:
|
||||||
elem = await self.actions.wait_for(
|
elem = await self.actions.wait_for(
|
||||||
"input_box", ctx.win_geom, timeout=2.0, interval=0.3
|
"input_box", ctx.win_geom, timeout=2.0, interval=0.3
|
||||||
@ -400,6 +458,8 @@ class SendTextFlow(Flow):
|
|||||||
"[flow:send_text] session_still_open OK (%.0fms)",
|
"[flow:send_text] session_still_open OK (%.0fms)",
|
||||||
(time.perf_counter() - t0) * 1000,
|
(time.perf_counter() - t0) * 1000,
|
||||||
)
|
)
|
||||||
|
except PermanentError:
|
||||||
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"[flow:send_text] session_still_open exception: %s (%.0fms)",
|
"[flow:send_text] session_still_open exception: %s (%.0fms)",
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user