Kris
3effe1338c
1. 新增WOC_DISABLE_OPENCV环境变量支持纯坐标定位模式 2. 优化xdotool文本输入：替换换行符为空格、添加30ms输入延迟 3. 新增输入框清空逻辑，避免内容残留
547 lines
21 KiB
Python
547 lines
21 KiB
Python
"""L1 xdotool/scrot 后端实现：XdotoolBackend。
|
||
|
||
通过 xdotool 调用 X11 自动化操作（点击 / 输入 / 按键 / 窗口管理），
|
||
scrot 截图输出 stdout 不落盘。所有子进程调用经 _run_managed 上下文管理器，
|
||
确保 TimeoutError / CancelledError / 其他异常路径下子进程均被 kill+wait 收尸，
|
||
无僵尸进程残留。
|
||
|
||
分层超时常量：
|
||
- _CMD_TIMEOUT_SEC=5.0：单条 xdotool/scrot 命令超时
|
||
- _STEP_MIN_TIMEOUT_SEC=7.0：单步 UI 操作最小超时
|
||
- _FLOW_TIMEOUT_SEC=30.0：完整发送流程超时
|
||
- _HTTP_TIMEOUT_SEC=60.0：HTTP 端到端超时
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import logging
|
||
import os
|
||
import time
|
||
from contextlib import asynccontextmanager
|
||
from typing import AsyncIterator, Optional
|
||
|
||
from woc_bridge.ui.backends.base import BackendProtocol, WindowGeometry
|
||
|
||
logger = logging.getLogger("woc-bridge")
|
||
|
||
# 分层超时常量（P1 阶段已落地，供 _run_managed / _step / flow / http 使用）
|
||
_CMD_TIMEOUT_SEC = 5.0
|
||
_STEP_MIN_TIMEOUT_SEC = 7.0
|
||
_FLOW_TIMEOUT_SEC = 30.0
|
||
_HTTP_TIMEOUT_SEC = 60.0
|
||
|
||
# xdotool type 每字符间隔（毫秒）：默认值 12ms 对中文 IME 过快，
|
||
# 容易导致字符丢失；提高到 30ms 给微信 UI 足够处理时间。
|
||
# 可通过环境变量 WOC_TYPE_DELAY_MS 覆盖。
|
||
_TYPE_DELAY_MS = int(os.environ.get("WOC_TYPE_DELAY_MS", "30"))
|
||
|
||
|
||
class XdotoolBackend(BackendProtocol):
|
||
"""xdotool + scrot 后端实现。
|
||
|
||
所有 UI 操作受 self._ui_lock 串行保护，避免并发点击导致焦点错乱。
|
||
asyncio.Lock 必须在事件循环中创建，故在 __init__ 中创建而非模块顶层。
|
||
"""
|
||
|
||
def __init__(self, display: str = ":1") -> None:
|
||
"""初始化后端。
|
||
|
||
Args:
|
||
display: X server display 地址，如 ":1"
|
||
"""
|
||
self.display = display
|
||
# asyncio.Lock 必须在事件循环中创建，不在模块顶层
|
||
self._ui_lock = asyncio.Lock()
|
||
self._debug_shots_enabled = (
|
||
os.environ.get("WOC_UI_DEBUG_SHOTS", "false").lower() == "true"
|
||
)
|
||
self._debug_queue: asyncio.Queue = asyncio.Queue()
|
||
self._debug_task: Optional[asyncio.Task] = None
|
||
# 主窗口 ID 缓存：window_title -> (window_id, expire_at)
|
||
# 避免 cache_hit 路径上 _find_main_window_id 被重复调用 3-5 次
|
||
# （activate_window + get_window_geometry + is_window_active 各调一次）
|
||
# TTL 5s，窗口切换/关闭时由下次查询自然失效
|
||
self._main_window_id_cache: dict[str, tuple[int, float]] = {}
|
||
self._main_window_id_cache_ttl = 5.0
|
||
|
||
# ------------------------------------------------------------------
|
||
# 环境与子进程管理
|
||
# ------------------------------------------------------------------
|
||
def _env(self) -> dict:
|
||
"""返回带 DISPLAY 的环境副本。"""
|
||
env = dict(os.environ)
|
||
env["DISPLAY"] = self.display
|
||
return env
|
||
|
||
@asynccontextmanager
|
||
async def _run_managed(
|
||
self, args: list[str]
|
||
) -> AsyncIterator[tuple[int, bytes, bytes]]:
|
||
"""执行子进程的上下文管理器，确保取消/超时/异常都 kill+wait 收尸。
|
||
|
||
用法：
|
||
async with self._run_managed(args) as (rc, stdout, stderr):
|
||
...
|
||
|
||
任何异常路径均保证 proc.kill() + await proc.wait()，无僵尸残留。
|
||
"""
|
||
proc = await asyncio.create_subprocess_exec(
|
||
*args,
|
||
stdout=asyncio.subprocess.PIPE,
|
||
stderr=asyncio.subprocess.PIPE,
|
||
env=self._env(),
|
||
)
|
||
try:
|
||
stdout, stderr = await asyncio.wait_for(
|
||
proc.communicate(), timeout=_CMD_TIMEOUT_SEC
|
||
)
|
||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||
# 显式 kill + wait，再超时仅记日志
|
||
proc.kill()
|
||
try:
|
||
await asyncio.wait_for(proc.wait(), timeout=5.0)
|
||
except asyncio.TimeoutError:
|
||
logger.error(
|
||
"[ui-backend] proc.wait() timeout after kill: %s", args[0]
|
||
)
|
||
raise
|
||
except Exception:
|
||
# 兜底：未退出的子进程一律 kill+wait（带超时，避免无限阻塞 _ui_lock）
|
||
if proc.returncode is None:
|
||
proc.kill()
|
||
try:
|
||
await asyncio.wait_for(proc.wait(), timeout=5.0)
|
||
except asyncio.TimeoutError:
|
||
logger.error(
|
||
"[ui-backend] proc.wait() timeout after kill: %s",
|
||
args[0] if args else "?",
|
||
)
|
||
raise
|
||
yield (proc.returncode, stdout, stderr)
|
||
|
||
async def _run(self, args: list[str]) -> tuple[int, bytes, bytes]:
|
||
"""执行一条命令并返回 (returncode, stdout, stderr)。"""
|
||
t0 = time.perf_counter()
|
||
async with self._run_managed(args) as result:
|
||
rc, stdout, stderr = result
|
||
# 隐藏 stdout/stderr 内容避免污染日志，仅记录长度和 rc
|
||
logger.debug(
|
||
"[ui-backend] cmd rc=%d %s (%.0fms) out=%dB err=%dB",
|
||
rc, " ".join(args[:6]),
|
||
(time.perf_counter() - t0) * 1000,
|
||
len(stdout) if stdout else 0,
|
||
len(stderr) if stderr else 0,
|
||
)
|
||
if rc != 0:
|
||
err_str = stderr.decode(errors="ignore").strip()[:200] if stderr else ""
|
||
logger.warning(
|
||
"[ui-backend] cmd FAILED rc=%d cmd=%s err=%s",
|
||
rc, " ".join(args), err_str,
|
||
)
|
||
return result
|
||
|
||
# ------------------------------------------------------------------
|
||
# BackendProtocol 实现
|
||
# ------------------------------------------------------------------
|
||
async def click(self, x: int, y: int) -> None:
|
||
"""合并 mousemove + click 1 为单条 xdotool 命令。
|
||
|
||
关键：单条命令，避免分两次 fork 之间窗口焦点变化导致点击错位。
|
||
|
||
不使用 --sync：mousemove --sync 会阻塞等待 X server 确认鼠标 motion
|
||
完成，在 VNC/Xvnc 环境下（pointer 被 grab、X server 不响应 motion
|
||
事件、窗口未真正激活等）会无限期死等，触发 5s 命令超时。去掉 --sync
|
||
与 activate_window 的处理保持一致（参见 L392 注释）。
|
||
xdotool 内部串行执行 mousemove → click，X server 按序处理 motion 与
|
||
button 事件，去掉 --sync 不影响点击精度。
|
||
"""
|
||
logger.info("[ui-backend] click start (%d,%d)", x, y)
|
||
t0 = time.perf_counter()
|
||
async with self._ui_lock:
|
||
await self._run(
|
||
[
|
||
"xdotool",
|
||
"mousemove",
|
||
str(x),
|
||
str(y),
|
||
"click",
|
||
"1",
|
||
]
|
||
)
|
||
logger.info(
|
||
"[ui-backend] click done (%d,%d) (%.0fms)",
|
||
x, y, (time.perf_counter() - t0) * 1000,
|
||
)
|
||
|
||
async def type_text(self, text: str) -> None:
|
||
"""xdotool type --delay <ms> -- <text>：原样输入文本，不解析修饰键组合。
|
||
|
||
使用 --delay 控制每字符间隔，避免中文 IME 处理不及导致字符丢失。
|
||
"""
|
||
text_len = len(text)
|
||
logger.info(
|
||
"[ui-backend] type_text start len=%d delay=%dms preview=%r",
|
||
text_len, _TYPE_DELAY_MS, text[:30],
|
||
)
|
||
t0 = time.perf_counter()
|
||
async with self._ui_lock:
|
||
await self._run(
|
||
[
|
||
"xdotool",
|
||
"type",
|
||
"--delay",
|
||
str(_TYPE_DELAY_MS),
|
||
"--",
|
||
text,
|
||
]
|
||
)
|
||
logger.info(
|
||
"[ui-backend] type_text done len=%d (%.0fms)",
|
||
text_len, (time.perf_counter() - t0) * 1000,
|
||
)
|
||
|
||
async def key_press(self, key: str, repeat: int = 1) -> None:
|
||
"""xdotool key [--repeat N] <key>。"""
|
||
logger.info("[ui-backend] key_press start key=%s repeat=%d", key, repeat)
|
||
t0 = time.perf_counter()
|
||
async with self._ui_lock:
|
||
args = ["xdotool", "key"]
|
||
if repeat > 1:
|
||
args.extend(["--repeat", str(repeat)])
|
||
args.append(key)
|
||
await self._run(args)
|
||
logger.info(
|
||
"[ui-backend] key_press done key=%s repeat=%d (%.0fms)",
|
||
key, repeat, (time.perf_counter() - t0) * 1000,
|
||
)
|
||
|
||
async def screenshot(self) -> bytes:
|
||
"""scrot 截图输出到临时文件，再读取为 bytes。
|
||
|
||
注意：s6 服务内 /dev/stdout 不是有效设备，scrot -o - 会失败，
|
||
故改用临时文件路径。超时 5s，TimeoutError/CancelledError 时 kill+wait。
|
||
"""
|
||
import tempfile
|
||
|
||
async with self._ui_lock:
|
||
fd, path = tempfile.mkstemp(suffix=".png", prefix="woc_screenshot_")
|
||
os.close(fd)
|
||
try:
|
||
proc = await asyncio.create_subprocess_exec(
|
||
"scrot",
|
||
"-o",
|
||
path,
|
||
stdout=asyncio.subprocess.PIPE,
|
||
stderr=asyncio.subprocess.PIPE,
|
||
env=self._env(),
|
||
)
|
||
try:
|
||
stdout, stderr = await asyncio.wait_for(
|
||
proc.communicate(), timeout=_CMD_TIMEOUT_SEC
|
||
)
|
||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||
proc.kill()
|
||
try:
|
||
await asyncio.wait_for(proc.wait(), timeout=5.0)
|
||
except asyncio.TimeoutError:
|
||
logger.error("[ui-backend] scrot proc.wait() timeout after kill")
|
||
raise
|
||
except Exception:
|
||
if proc.returncode is None:
|
||
proc.kill()
|
||
try:
|
||
await asyncio.wait_for(proc.wait(), timeout=5.0)
|
||
except asyncio.TimeoutError:
|
||
logger.error(
|
||
"[ui-backend] scrot proc.wait() timeout after kill"
|
||
)
|
||
raise
|
||
|
||
if proc.returncode != 0:
|
||
err = stderr.decode(errors="ignore").strip()[:200]
|
||
logger.error("[ui-backend] screenshot failed: rc=%s err=%s", proc.returncode, err)
|
||
return b""
|
||
|
||
with open(path, "rb") as f:
|
||
data = f.read()
|
||
logger.info(
|
||
"[ui-backend] screenshot: scrot rc=0 file=%s bytes=%d",
|
||
path, len(data),
|
||
)
|
||
return data
|
||
finally:
|
||
try:
|
||
os.unlink(path)
|
||
except OSError:
|
||
pass
|
||
|
||
async def _find_main_window_id(
|
||
self, window_title: str, min_area: int = 10000
|
||
) -> int:
|
||
"""查找窗口标题匹配者中面积最大的主窗口，过滤隐藏/加载小窗。
|
||
|
||
微信启动过程中会出现多个窗口（隐藏加载窗、托盘提示窗等），
|
||
仅按标题搜索并取第一个常常会拿到 1×1 的无效窗口。本方法：
|
||
1. 搜索所有标题匹配窗口；
|
||
2. 逐个获取几何并计算面积；
|
||
3. 过滤掉面积 < min_area 的候选；
|
||
4. 返回面积最大者。
|
||
|
||
带 5s TTL 缓存：cache_hit 路径上 activate_window + get_window_geometry
|
||
+ is_window_active 会重复调用本方法，缓存避免 3-5 次 xdotool search。
|
||
|
||
Args:
|
||
window_title: 窗口标题，如 "微信"
|
||
min_area: 最小有效窗口面积（默认 10000 像素）
|
||
|
||
Returns:
|
||
主窗口 ID
|
||
|
||
Raises:
|
||
RuntimeError: 未找到有效窗口
|
||
"""
|
||
# 缓存命中检查
|
||
cached = self._main_window_id_cache.get(window_title)
|
||
if cached is not None:
|
||
wid, expire_at = cached
|
||
if time.monotonic() < expire_at:
|
||
return wid
|
||
self._main_window_id_cache.pop(window_title, None)
|
||
|
||
rc, stdout, _ = await self._run(
|
||
["xdotool", "search", "--name", window_title]
|
||
)
|
||
if rc != 0:
|
||
raise RuntimeError(
|
||
f"WINDOW_NOT_FOUND: search failed for title={window_title!r}"
|
||
)
|
||
text = stdout.decode(errors="ignore").strip()
|
||
if not text:
|
||
raise RuntimeError(
|
||
f"WINDOW_NOT_FOUND: no window matches title={window_title!r}"
|
||
)
|
||
|
||
candidates: list[tuple[int, int]] = []
|
||
for line in text.splitlines():
|
||
wid_str = line.strip()
|
||
if not wid_str:
|
||
continue
|
||
try:
|
||
wid = int(wid_str)
|
||
except ValueError:
|
||
continue
|
||
rc, gstdout, _ = await self._run(
|
||
["xdotool", "getwindowgeometry", "--shell", str(wid)]
|
||
)
|
||
if rc != 0:
|
||
continue
|
||
geom: dict[str, str] = {}
|
||
for gline in gstdout.decode(errors="ignore").splitlines():
|
||
if "=" in gline:
|
||
k, v = gline.split("=", 1)
|
||
geom[k.strip()] = v.strip()
|
||
try:
|
||
w = int(geom.get("WIDTH", 0))
|
||
h = int(geom.get("HEIGHT", 0))
|
||
area = w * h
|
||
if area >= min_area and w >= 200 and h >= 200:
|
||
candidates.append((area, wid))
|
||
except ValueError:
|
||
continue
|
||
|
||
if not candidates:
|
||
raise RuntimeError(
|
||
f"WINDOW_NOT_FOUND: no valid window for title={window_title!r} "
|
||
f"(candidates filtered by min_area={min_area})"
|
||
)
|
||
candidates.sort(key=lambda x: x[0], reverse=True)
|
||
main_id = candidates[0][1]
|
||
# 写入缓存
|
||
self._main_window_id_cache[window_title] = (
|
||
main_id, time.monotonic() + self._main_window_id_cache_ttl
|
||
)
|
||
return main_id
|
||
|
||
async def get_window_geometry(self, window_title: str) -> WindowGeometry:
|
||
"""按标题查窗口并返回几何。容忍多个匹配窗口，取面积最大主窗口。
|
||
|
||
Raises:
|
||
RuntimeError: 窗口未找到（TODO: P3 时换 FlowError(code=WINDOW_NOT_FOUND)）
|
||
"""
|
||
t0 = time.perf_counter()
|
||
logger.info("[ui-backend] get_window_geometry start title=%r", window_title)
|
||
try:
|
||
window_id = await self._find_main_window_id(window_title)
|
||
except RuntimeError as exc:
|
||
logger.warning(
|
||
"[ui-backend] get_window_geometry: %s (%.0fms)",
|
||
exc, (time.perf_counter() - t0) * 1000,
|
||
)
|
||
raise
|
||
|
||
rc, stdout, _ = await self._run(
|
||
["xdotool", "getwindowgeometry", "--shell", str(window_id)]
|
||
)
|
||
if rc != 0:
|
||
logger.warning(
|
||
"[ui-backend] get_window_geometry: getwindowgeometry failed id=%d",
|
||
window_id,
|
||
)
|
||
raise RuntimeError(
|
||
f"WINDOW_NOT_FOUND: getwindowgeometry failed for id={window_id}"
|
||
)
|
||
geom: dict[str, str] = {}
|
||
for line in stdout.decode(errors="ignore").splitlines():
|
||
if "=" in line:
|
||
k, v = line.split("=", 1)
|
||
geom[k.strip()] = v.strip()
|
||
try:
|
||
result = WindowGeometry(
|
||
x=int(geom.get("X", 0)),
|
||
y=int(geom.get("Y", 0)),
|
||
width=int(geom.get("WIDTH", 0)),
|
||
height=int(geom.get("HEIGHT", 0)),
|
||
)
|
||
logger.info(
|
||
"[ui-backend] get_window_geometry OK id=%d geom=%dx%d@(%d,%d) (%.0fms)",
|
||
window_id, result.width, result.height, result.x, result.y,
|
||
(time.perf_counter() - t0) * 1000,
|
||
)
|
||
return result
|
||
except ValueError:
|
||
logger.warning(
|
||
"[ui-backend] get_window_geometry: invalid geom dict=%s id=%d",
|
||
geom, window_id,
|
||
)
|
||
raise RuntimeError(
|
||
f"WINDOW_NOT_FOUND: invalid geometry for id={window_id}"
|
||
)
|
||
|
||
async def activate_window(self, window_title: str) -> None:
|
||
"""激活指定标题的窗口。
|
||
|
||
Raises:
|
||
RuntimeError: 窗口未找到（TODO: P3 时换 FlowError(code=WINDOW_NOT_FOUND)）
|
||
"""
|
||
t0 = time.perf_counter()
|
||
logger.info("[ui-backend] activate_window start title=%r", window_title)
|
||
try:
|
||
window_id = await self._find_main_window_id(window_title)
|
||
except RuntimeError as exc:
|
||
logger.warning(
|
||
"[ui-backend] activate_window: %s (%.0fms)",
|
||
exc, (time.perf_counter() - t0) * 1000,
|
||
)
|
||
raise
|
||
# 使用非阻塞 windowactivate，避免 --sync 在 VNC 无人操作时死等。
|
||
# 之后轮询 getactivewindow 确认焦点是否真正落到目标窗口。
|
||
rc, _, stderr = await self._run(
|
||
["xdotool", "windowactivate", str(window_id)]
|
||
)
|
||
if rc != 0:
|
||
err_str = stderr.decode(errors="ignore").strip()[:200] if stderr else ""
|
||
logger.warning(
|
||
"[ui-backend] activate_window: windowactivate failed id=%d rc=%d err=%s",
|
||
window_id, rc, err_str,
|
||
)
|
||
raise RuntimeError(
|
||
f"WINDOW_ACTIVATE_FAILED: windowactivate failed for id={window_id} rc={rc} err={err_str}"
|
||
)
|
||
|
||
deadline = time.monotonic() + 2.0
|
||
activated = False
|
||
while time.monotonic() < deadline:
|
||
rc, stdout, _ = await self._run(["xdotool", "getactivewindow"])
|
||
if rc == 0:
|
||
active_text = stdout.decode(errors="ignore").strip()
|
||
try:
|
||
if int(active_text) == window_id:
|
||
activated = True
|
||
break
|
||
except ValueError:
|
||
pass
|
||
await asyncio.sleep(0.1)
|
||
|
||
if activated:
|
||
logger.info(
|
||
"[ui-backend] activate_window OK id=%d title=%r (%.0fms)",
|
||
window_id, window_title, (time.perf_counter() - t0) * 1000,
|
||
)
|
||
else:
|
||
logger.warning(
|
||
"[ui-backend] activate_window: focus verify timeout id=%d title=%r (%.0fms), continue",
|
||
window_id, window_title, (time.perf_counter() - t0) * 1000,
|
||
)
|
||
|
||
async def is_window_active(self, window_title: str) -> bool:
|
||
"""校验指定标题的窗口是否是当前活动窗口。
|
||
|
||
用于纯坐标模式下 _verify_session_still_open 校验窗口焦点：
|
||
activate_window 调用后焦点校验可能超时（只记 warning 不 raise），
|
||
此方法提供二次校验，若窗口已失焦则 cache 失效走完整流程。
|
||
|
||
Returns:
|
||
True 表示窗口存在且是活动窗口；False 表示窗口不存在或非活动窗口
|
||
"""
|
||
try:
|
||
rc, stdout, _ = await self._run(["xdotool", "getactivewindow"])
|
||
if rc != 0:
|
||
return False
|
||
active_text = stdout.decode(errors="ignore").strip()
|
||
try:
|
||
active_id = int(active_text)
|
||
except ValueError:
|
||
return False
|
||
main_id = await self._find_main_window_id(window_title)
|
||
return active_id == main_id
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"[ui-backend] is_window_active exception title=%r: %s",
|
||
window_title, exc,
|
||
)
|
||
return False
|
||
|
||
# ------------------------------------------------------------------
|
||
# 调试截图异步队列（生产环境关闭，调试模式后台写盘）
|
||
# ------------------------------------------------------------------
|
||
async def _debug_screenshot_async(self, step: str, trace_id: str = "") -> None:
|
||
"""入队调试截图请求，由后台 worker 串行写盘，不阻塞关键路径。
|
||
|
||
生产环境（WOC_UI_DEBUG_SHOTS=false）直接 return。
|
||
"""
|
||
if not self._debug_shots_enabled:
|
||
return
|
||
try:
|
||
self._debug_queue.put_nowait((step, trace_id, time.time()))
|
||
except asyncio.QueueFull:
|
||
logger.warning("[ui-backend] debug queue full, drop shot step=%s", step)
|
||
return
|
||
if self._debug_task is None or self._debug_task.done():
|
||
self._debug_task = asyncio.create_task(self._debug_writer())
|
||
|
||
async def _debug_writer(self) -> None:
|
||
"""后台 worker：从队列取 (step, trace_id, ts) 写盘到 /tmp/woc_debug/。
|
||
|
||
文件名格式：{trace_id}_{step}_{ts_ms}.png
|
||
异常时记 warning 不抛，避免拖垮整个后端。
|
||
"""
|
||
while True:
|
||
try:
|
||
step, trace_id, ts = await self._debug_queue.get()
|
||
except asyncio.CancelledError:
|
||
return
|
||
try:
|
||
os.makedirs("/tmp/woc_debug", exist_ok=True)
|
||
path = f"/tmp/woc_debug/{trace_id}_{step}_{int(ts * 1000)}.png"
|
||
await self._run(["scrot", path])
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"[ui-backend] debug writer failed step=%s: %s", step, exc
|
||
)
|
||
|
||
@property
|
||
def capabilities(self) -> set[str]:
|
||
"""返回后端支持的能力集合。"""
|
||
return {"click", "type", "key", "screenshot", "window"}
|