"""L1 xdotool/scrot 后端实现:XdotoolBackend。 通过 xdotool 调用 X11 自动化操作(点击 / 输入 / 按键 / 窗口管理), scrot 截图输出 stdout 不落盘。所有子进程调用经 _run_managed 上下文管理器, 确保 TimeoutError / CancelledError / 其他异常路径下子进程均被 kill+wait 收尸, 无僵尸进程残留。 分层超时常量: - _CMD_TIMEOUT_SEC=5.0:单条 xdotool/scrot 命令超时 - _STEP_MIN_TIMEOUT_SEC=7.0:单步 UI 操作最小超时 - _FLOW_TIMEOUT_SEC=30.0:完整发送流程超时 - _HTTP_TIMEOUT_SEC=60.0:HTTP 端到端超时 """ from __future__ import annotations import asyncio import logging import os import time from contextlib import asynccontextmanager from typing import AsyncIterator, Optional from woc_bridge.ui.backends.base import BackendProtocol, WindowGeometry logger = logging.getLogger("woc-bridge") # 分层超时常量(P1 阶段已落地,供 _run_managed / _step / flow / http 使用) _CMD_TIMEOUT_SEC = 5.0 _STEP_MIN_TIMEOUT_SEC = 7.0 _FLOW_TIMEOUT_SEC = 30.0 _HTTP_TIMEOUT_SEC = 60.0 # xdotool type 每字符间隔(毫秒):默认值 12ms 对中文 IME 过快, # 容易导致字符丢失;提高到 30ms 给微信 UI 足够处理时间。 # 可通过环境变量 WOC_TYPE_DELAY_MS 覆盖。 _TYPE_DELAY_MS = int(os.environ.get("WOC_TYPE_DELAY_MS", "30")) class XdotoolBackend(BackendProtocol): """xdotool + scrot 后端实现。 所有 UI 操作受 self._ui_lock 串行保护,避免并发点击导致焦点错乱。 asyncio.Lock 必须在事件循环中创建,故在 __init__ 中创建而非模块顶层。 """ def __init__(self, display: str = ":1") -> None: """初始化后端。 Args: display: X server display 地址,如 ":1" """ self.display = display # asyncio.Lock 必须在事件循环中创建,不在模块顶层 self._ui_lock = asyncio.Lock() self._debug_shots_enabled = ( os.environ.get("WOC_UI_DEBUG_SHOTS", "false").lower() == "true" ) self._debug_queue: asyncio.Queue = asyncio.Queue() self._debug_task: Optional[asyncio.Task] = None # 主窗口 ID 缓存:window_title -> (window_id, expire_at) # 避免 cache_hit 路径上 _find_main_window_id 被重复调用 3-5 次 # (activate_window + get_window_geometry + is_window_active 各调一次) # TTL 5s,窗口切换/关闭时由下次查询自然失效 self._main_window_id_cache: dict[str, tuple[int, float]] = {} self._main_window_id_cache_ttl = 5.0 # ------------------------------------------------------------------ # 环境与子进程管理 # ------------------------------------------------------------------ def _env(self) -> dict: """返回带 DISPLAY 的环境副本。""" env = dict(os.environ) env["DISPLAY"] = self.display return env @asynccontextmanager async def _run_managed( self, args: list[str] ) -> AsyncIterator[tuple[int, bytes, bytes]]: """执行子进程的上下文管理器,确保取消/超时/异常都 kill+wait 收尸。 用法: async with self._run_managed(args) as (rc, stdout, stderr): ... 任何异常路径均保证 proc.kill() + await proc.wait(),无僵尸残留。 """ proc = await asyncio.create_subprocess_exec( *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=self._env(), ) try: stdout, stderr = await asyncio.wait_for( proc.communicate(), timeout=_CMD_TIMEOUT_SEC ) except (asyncio.TimeoutError, asyncio.CancelledError): # 显式 kill + wait,再超时仅记日志 proc.kill() try: await asyncio.wait_for(proc.wait(), timeout=5.0) except asyncio.TimeoutError: logger.error( "[ui-backend] proc.wait() timeout after kill: %s", args[0] ) raise except Exception: # 兜底:未退出的子进程一律 kill+wait(带超时,避免无限阻塞 _ui_lock) if proc.returncode is None: proc.kill() try: await asyncio.wait_for(proc.wait(), timeout=5.0) except asyncio.TimeoutError: logger.error( "[ui-backend] proc.wait() timeout after kill: %s", args[0] if args else "?", ) raise yield (proc.returncode, stdout, stderr) async def _run(self, args: list[str]) -> tuple[int, bytes, bytes]: """执行一条命令并返回 (returncode, stdout, stderr)。""" t0 = time.perf_counter() async with self._run_managed(args) as result: rc, stdout, stderr = result # 隐藏 stdout/stderr 内容避免污染日志,仅记录长度和 rc logger.debug( "[ui-backend] cmd rc=%d %s (%.0fms) out=%dB err=%dB", rc, " ".join(args[:6]), (time.perf_counter() - t0) * 1000, len(stdout) if stdout else 0, len(stderr) if stderr else 0, ) if rc != 0: err_str = stderr.decode(errors="ignore").strip()[:200] if stderr else "" logger.warning( "[ui-backend] cmd FAILED rc=%d cmd=%s err=%s", rc, " ".join(args), err_str, ) return result # ------------------------------------------------------------------ # BackendProtocol 实现 # ------------------------------------------------------------------ async def click(self, x: int, y: int) -> None: """合并 mousemove + click 1 为单条 xdotool 命令。 关键:单条命令,避免分两次 fork 之间窗口焦点变化导致点击错位。 不使用 --sync:mousemove --sync 会阻塞等待 X server 确认鼠标 motion 完成,在 VNC/Xvnc 环境下(pointer 被 grab、X server 不响应 motion 事件、窗口未真正激活等)会无限期死等,触发 5s 命令超时。去掉 --sync 与 activate_window 的处理保持一致(参见 L392 注释)。 xdotool 内部串行执行 mousemove → click,X server 按序处理 motion 与 button 事件,去掉 --sync 不影响点击精度。 """ logger.info("[ui-backend] click start (%d,%d)", x, y) t0 = time.perf_counter() async with self._ui_lock: await self._run( [ "xdotool", "mousemove", str(x), str(y), "click", "1", ] ) logger.info( "[ui-backend] click done (%d,%d) (%.0fms)", x, y, (time.perf_counter() - t0) * 1000, ) async def type_text(self, text: str) -> None: """xdotool type --delay -- :原样输入文本,不解析修饰键组合。 使用 --delay 控制每字符间隔,避免中文 IME 处理不及导致字符丢失。 """ text_len = len(text) logger.info( "[ui-backend] type_text start len=%d delay=%dms preview=%r", text_len, _TYPE_DELAY_MS, text[:30], ) t0 = time.perf_counter() async with self._ui_lock: await self._run( [ "xdotool", "type", "--delay", str(_TYPE_DELAY_MS), "--", text, ] ) logger.info( "[ui-backend] type_text done len=%d (%.0fms)", text_len, (time.perf_counter() - t0) * 1000, ) async def key_press(self, key: str, repeat: int = 1) -> None: """xdotool key [--repeat N] 。""" logger.info("[ui-backend] key_press start key=%s repeat=%d", key, repeat) t0 = time.perf_counter() async with self._ui_lock: args = ["xdotool", "key"] if repeat > 1: args.extend(["--repeat", str(repeat)]) args.append(key) await self._run(args) logger.info( "[ui-backend] key_press done key=%s repeat=%d (%.0fms)", key, repeat, (time.perf_counter() - t0) * 1000, ) async def screenshot(self) -> bytes: """scrot 截图输出到临时文件,再读取为 bytes。 注意:s6 服务内 /dev/stdout 不是有效设备,scrot -o - 会失败, 故改用临时文件路径。超时 5s,TimeoutError/CancelledError 时 kill+wait。 """ import tempfile async with self._ui_lock: fd, path = tempfile.mkstemp(suffix=".png", prefix="woc_screenshot_") os.close(fd) try: proc = await asyncio.create_subprocess_exec( "scrot", "-o", path, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=self._env(), ) try: stdout, stderr = await asyncio.wait_for( proc.communicate(), timeout=_CMD_TIMEOUT_SEC ) except (asyncio.TimeoutError, asyncio.CancelledError): proc.kill() try: await asyncio.wait_for(proc.wait(), timeout=5.0) except asyncio.TimeoutError: logger.error("[ui-backend] scrot proc.wait() timeout after kill") raise except Exception: if proc.returncode is None: proc.kill() try: await asyncio.wait_for(proc.wait(), timeout=5.0) except asyncio.TimeoutError: logger.error( "[ui-backend] scrot proc.wait() timeout after kill" ) raise if proc.returncode != 0: err = stderr.decode(errors="ignore").strip()[:200] logger.error("[ui-backend] screenshot failed: rc=%s err=%s", proc.returncode, err) return b"" with open(path, "rb") as f: data = f.read() logger.info( "[ui-backend] screenshot: scrot rc=0 file=%s bytes=%d", path, len(data), ) return data finally: try: os.unlink(path) except OSError: pass async def _find_main_window_id( self, window_title: str, min_area: int = 10000 ) -> int: """查找窗口标题匹配者中面积最大的主窗口,过滤隐藏/加载小窗。 微信启动过程中会出现多个窗口(隐藏加载窗、托盘提示窗等), 仅按标题搜索并取第一个常常会拿到 1×1 的无效窗口。本方法: 1. 搜索所有标题匹配窗口; 2. 逐个获取几何并计算面积; 3. 过滤掉面积 < min_area 的候选; 4. 返回面积最大者。 带 5s TTL 缓存:cache_hit 路径上 activate_window + get_window_geometry + is_window_active 会重复调用本方法,缓存避免 3-5 次 xdotool search。 Args: window_title: 窗口标题,如 "微信" min_area: 最小有效窗口面积(默认 10000 像素) Returns: 主窗口 ID Raises: RuntimeError: 未找到有效窗口 """ # 缓存命中检查 cached = self._main_window_id_cache.get(window_title) if cached is not None: wid, expire_at = cached if time.monotonic() < expire_at: return wid self._main_window_id_cache.pop(window_title, None) rc, stdout, _ = await self._run( ["xdotool", "search", "--name", window_title] ) if rc != 0: raise RuntimeError( f"WINDOW_NOT_FOUND: search failed for title={window_title!r}" ) text = stdout.decode(errors="ignore").strip() if not text: raise RuntimeError( f"WINDOW_NOT_FOUND: no window matches title={window_title!r}" ) candidates: list[tuple[int, int]] = [] for line in text.splitlines(): wid_str = line.strip() if not wid_str: continue try: wid = int(wid_str) except ValueError: continue rc, gstdout, _ = await self._run( ["xdotool", "getwindowgeometry", "--shell", str(wid)] ) if rc != 0: continue geom: dict[str, str] = {} for gline in gstdout.decode(errors="ignore").splitlines(): if "=" in gline: k, v = gline.split("=", 1) geom[k.strip()] = v.strip() try: w = int(geom.get("WIDTH", 0)) h = int(geom.get("HEIGHT", 0)) area = w * h if area >= min_area and w >= 200 and h >= 200: candidates.append((area, wid)) except ValueError: continue if not candidates: raise RuntimeError( f"WINDOW_NOT_FOUND: no valid window for title={window_title!r} " f"(candidates filtered by min_area={min_area})" ) candidates.sort(key=lambda x: x[0], reverse=True) main_id = candidates[0][1] # 写入缓存 self._main_window_id_cache[window_title] = ( main_id, time.monotonic() + self._main_window_id_cache_ttl ) return main_id async def get_window_geometry(self, window_title: str) -> WindowGeometry: """按标题查窗口并返回几何。容忍多个匹配窗口,取面积最大主窗口。 Raises: RuntimeError: 窗口未找到(TODO: P3 时换 FlowError(code=WINDOW_NOT_FOUND)) """ t0 = time.perf_counter() logger.info("[ui-backend] get_window_geometry start title=%r", window_title) try: window_id = await self._find_main_window_id(window_title) except RuntimeError as exc: logger.warning( "[ui-backend] get_window_geometry: %s (%.0fms)", exc, (time.perf_counter() - t0) * 1000, ) raise rc, stdout, _ = await self._run( ["xdotool", "getwindowgeometry", "--shell", str(window_id)] ) if rc != 0: logger.warning( "[ui-backend] get_window_geometry: getwindowgeometry failed id=%d", window_id, ) raise RuntimeError( f"WINDOW_NOT_FOUND: getwindowgeometry failed for id={window_id}" ) geom: dict[str, str] = {} for line in stdout.decode(errors="ignore").splitlines(): if "=" in line: k, v = line.split("=", 1) geom[k.strip()] = v.strip() try: result = WindowGeometry( x=int(geom.get("X", 0)), y=int(geom.get("Y", 0)), width=int(geom.get("WIDTH", 0)), height=int(geom.get("HEIGHT", 0)), ) logger.info( "[ui-backend] get_window_geometry OK id=%d geom=%dx%d@(%d,%d) (%.0fms)", window_id, result.width, result.height, result.x, result.y, (time.perf_counter() - t0) * 1000, ) return result except ValueError: logger.warning( "[ui-backend] get_window_geometry: invalid geom dict=%s id=%d", geom, window_id, ) raise RuntimeError( f"WINDOW_NOT_FOUND: invalid geometry for id={window_id}" ) async def activate_window(self, window_title: str) -> None: """激活指定标题的窗口。 Raises: RuntimeError: 窗口未找到(TODO: P3 时换 FlowError(code=WINDOW_NOT_FOUND)) """ t0 = time.perf_counter() logger.info("[ui-backend] activate_window start title=%r", window_title) try: window_id = await self._find_main_window_id(window_title) except RuntimeError as exc: logger.warning( "[ui-backend] activate_window: %s (%.0fms)", exc, (time.perf_counter() - t0) * 1000, ) raise # 使用非阻塞 windowactivate,避免 --sync 在 VNC 无人操作时死等。 # 之后轮询 getactivewindow 确认焦点是否真正落到目标窗口。 rc, _, stderr = await self._run( ["xdotool", "windowactivate", str(window_id)] ) if rc != 0: err_str = stderr.decode(errors="ignore").strip()[:200] if stderr else "" logger.warning( "[ui-backend] activate_window: windowactivate failed id=%d rc=%d err=%s", window_id, rc, err_str, ) raise RuntimeError( f"WINDOW_ACTIVATE_FAILED: windowactivate failed for id={window_id} rc={rc} err={err_str}" ) deadline = time.monotonic() + 2.0 activated = False while time.monotonic() < deadline: rc, stdout, _ = await self._run(["xdotool", "getactivewindow"]) if rc == 0: active_text = stdout.decode(errors="ignore").strip() try: if int(active_text) == window_id: activated = True break except ValueError: pass await asyncio.sleep(0.1) if activated: logger.info( "[ui-backend] activate_window OK id=%d title=%r (%.0fms)", window_id, window_title, (time.perf_counter() - t0) * 1000, ) else: logger.warning( "[ui-backend] activate_window: focus verify timeout id=%d title=%r (%.0fms), continue", window_id, window_title, (time.perf_counter() - t0) * 1000, ) async def is_window_active(self, window_title: str) -> bool: """校验指定标题的窗口是否是当前活动窗口。 用于纯坐标模式下 _verify_session_still_open 校验窗口焦点: activate_window 调用后焦点校验可能超时(只记 warning 不 raise), 此方法提供二次校验,若窗口已失焦则 cache 失效走完整流程。 Returns: True 表示窗口存在且是活动窗口;False 表示窗口不存在或非活动窗口 """ try: rc, stdout, _ = await self._run(["xdotool", "getactivewindow"]) if rc != 0: return False active_text = stdout.decode(errors="ignore").strip() try: active_id = int(active_text) except ValueError: return False main_id = await self._find_main_window_id(window_title) return active_id == main_id except Exception as exc: logger.warning( "[ui-backend] is_window_active exception title=%r: %s", window_title, exc, ) return False # ------------------------------------------------------------------ # 调试截图异步队列(生产环境关闭,调试模式后台写盘) # ------------------------------------------------------------------ async def _debug_screenshot_async(self, step: str, trace_id: str = "") -> None: """入队调试截图请求,由后台 worker 串行写盘,不阻塞关键路径。 生产环境(WOC_UI_DEBUG_SHOTS=false)直接 return。 """ if not self._debug_shots_enabled: return try: self._debug_queue.put_nowait((step, trace_id, time.time())) except asyncio.QueueFull: logger.warning("[ui-backend] debug queue full, drop shot step=%s", step) return if self._debug_task is None or self._debug_task.done(): self._debug_task = asyncio.create_task(self._debug_writer()) async def _debug_writer(self) -> None: """后台 worker:从队列取 (step, trace_id, ts) 写盘到 /tmp/woc_debug/。 文件名格式:{trace_id}_{step}_{ts_ms}.png 异常时记 warning 不抛,避免拖垮整个后端。 """ while True: try: step, trace_id, ts = await self._debug_queue.get() except asyncio.CancelledError: return try: os.makedirs("/tmp/woc_debug", exist_ok=True) path = f"/tmp/woc_debug/{trace_id}_{step}_{int(ts * 1000)}.png" await self._run(["scrot", path]) except Exception as exc: logger.warning( "[ui-backend] debug writer failed step=%s: %s", step, exc ) @property def capabilities(self) -> set[str]: """返回后端支持的能力集合。""" return {"click", "type", "key", "screenshot", "window"}