WechatOnCloud/bridge/woc_bridge/ui/backends/xdotool.py

547 lines
21 KiB
Python
Raw Normal View History

"""L1 xdotool/scrot 后端实现XdotoolBackend。
通过 xdotool 调用 X11 自动化操作点击 / 输入 / 按键 / 窗口管理
scrot 截图输出 stdout 不落盘所有子进程调用经 _run_managed 上下文管理器
确保 TimeoutError / CancelledError / 其他异常路径下子进程均被 kill+wait 收尸
无僵尸进程残留
分层超时常量
- _CMD_TIMEOUT_SEC=5.0单条 xdotool/scrot 命令超时
- _STEP_MIN_TIMEOUT_SEC=7.0单步 UI 操作最小超时
- _FLOW_TIMEOUT_SEC=30.0完整发送流程超时
- _HTTP_TIMEOUT_SEC=60.0HTTP 端到端超时
"""
from __future__ import annotations
import asyncio
import logging
import os
import time
from contextlib import asynccontextmanager
from typing import AsyncIterator, Optional
from woc_bridge.ui.backends.base import BackendProtocol, WindowGeometry
logger = logging.getLogger("woc-bridge")
# 分层超时常量P1 阶段已落地,供 _run_managed / _step / flow / http 使用)
_CMD_TIMEOUT_SEC = 5.0
_STEP_MIN_TIMEOUT_SEC = 7.0
_FLOW_TIMEOUT_SEC = 30.0
_HTTP_TIMEOUT_SEC = 60.0
# xdotool type 每字符间隔(毫秒):默认值 12ms 对中文 IME 过快,
# 容易导致字符丢失;提高到 30ms 给微信 UI 足够处理时间。
# 可通过环境变量 WOC_TYPE_DELAY_MS 覆盖。
_TYPE_DELAY_MS = int(os.environ.get("WOC_TYPE_DELAY_MS", "30"))
class XdotoolBackend(BackendProtocol):
"""xdotool + scrot 后端实现。
所有 UI 操作受 self._ui_lock 串行保护避免并发点击导致焦点错乱
asyncio.Lock 必须在事件循环中创建故在 __init__ 中创建而非模块顶层
"""
def __init__(self, display: str = ":1") -> None:
"""初始化后端。
Args:
display: X server display 地址 ":1"
"""
self.display = display
# asyncio.Lock 必须在事件循环中创建,不在模块顶层
self._ui_lock = asyncio.Lock()
self._debug_shots_enabled = (
os.environ.get("WOC_UI_DEBUG_SHOTS", "false").lower() == "true"
)
self._debug_queue: asyncio.Queue = asyncio.Queue()
self._debug_task: Optional[asyncio.Task] = None
# 主窗口 ID 缓存window_title -> (window_id, expire_at)
# 避免 cache_hit 路径上 _find_main_window_id 被重复调用 3-5 次
# activate_window + get_window_geometry + is_window_active 各调一次)
# TTL 5s窗口切换/关闭时由下次查询自然失效
self._main_window_id_cache: dict[str, tuple[int, float]] = {}
self._main_window_id_cache_ttl = 5.0
# ------------------------------------------------------------------
# 环境与子进程管理
# ------------------------------------------------------------------
def _env(self) -> dict:
"""返回带 DISPLAY 的环境副本。"""
env = dict(os.environ)
env["DISPLAY"] = self.display
return env
@asynccontextmanager
async def _run_managed(
self, args: list[str]
) -> AsyncIterator[tuple[int, bytes, bytes]]:
"""执行子进程的上下文管理器,确保取消/超时/异常都 kill+wait 收尸。
用法
async with self._run_managed(args) as (rc, stdout, stderr):
...
任何异常路径均保证 proc.kill() + await proc.wait()无僵尸残留
"""
proc = await asyncio.create_subprocess_exec(
*args,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
env=self._env(),
)
try:
stdout, stderr = await asyncio.wait_for(
proc.communicate(), timeout=_CMD_TIMEOUT_SEC
)
except (asyncio.TimeoutError, asyncio.CancelledError):
# 显式 kill + wait再超时仅记日志
proc.kill()
try:
await asyncio.wait_for(proc.wait(), timeout=5.0)
except asyncio.TimeoutError:
logger.error(
"[ui-backend] proc.wait() timeout after kill: %s", args[0]
)
raise
except Exception:
# 兜底:未退出的子进程一律 kill+wait带超时避免无限阻塞 _ui_lock
if proc.returncode is None:
proc.kill()
try:
await asyncio.wait_for(proc.wait(), timeout=5.0)
except asyncio.TimeoutError:
logger.error(
"[ui-backend] proc.wait() timeout after kill: %s",
args[0] if args else "?",
)
raise
yield (proc.returncode, stdout, stderr)
async def _run(self, args: list[str]) -> tuple[int, bytes, bytes]:
"""执行一条命令并返回 (returncode, stdout, stderr)。"""
t0 = time.perf_counter()
async with self._run_managed(args) as result:
rc, stdout, stderr = result
# 隐藏 stdout/stderr 内容避免污染日志,仅记录长度和 rc
logger.debug(
"[ui-backend] cmd rc=%d %s (%.0fms) out=%dB err=%dB",
rc, " ".join(args[:6]),
(time.perf_counter() - t0) * 1000,
len(stdout) if stdout else 0,
len(stderr) if stderr else 0,
)
if rc != 0:
err_str = stderr.decode(errors="ignore").strip()[:200] if stderr else ""
logger.warning(
"[ui-backend] cmd FAILED rc=%d cmd=%s err=%s",
rc, " ".join(args), err_str,
)
return result
# ------------------------------------------------------------------
# BackendProtocol 实现
# ------------------------------------------------------------------
async def click(self, x: int, y: int) -> None:
"""合并 mousemove + click 1 为单条 xdotool 命令。
关键单条命令避免分两次 fork 之间窗口焦点变化导致点击错位
不使用 --syncmousemove --sync 会阻塞等待 X server 确认鼠标 motion
完成 VNC/Xvnc 环境下pointer grabX server 不响应 motion
事件窗口未真正激活等会无限期死等触发 5s 命令超时去掉 --sync
activate_window 的处理保持一致参见 L392 注释
xdotool 内部串行执行 mousemove clickX server 按序处理 motion
button 事件去掉 --sync 不影响点击精度
"""
logger.info("[ui-backend] click start (%d,%d)", x, y)
t0 = time.perf_counter()
async with self._ui_lock:
await self._run(
[
"xdotool",
"mousemove",
str(x),
str(y),
"click",
"1",
]
)
logger.info(
"[ui-backend] click done (%d,%d) (%.0fms)",
x, y, (time.perf_counter() - t0) * 1000,
)
async def type_text(self, text: str) -> None:
"""xdotool type --delay <ms> -- <text>:原样输入文本,不解析修饰键组合。
使用 --delay 控制每字符间隔避免中文 IME 处理不及导致字符丢失
"""
text_len = len(text)
logger.info(
"[ui-backend] type_text start len=%d delay=%dms preview=%r",
text_len, _TYPE_DELAY_MS, text[:30],
)
t0 = time.perf_counter()
async with self._ui_lock:
await self._run(
[
"xdotool",
"type",
"--delay",
str(_TYPE_DELAY_MS),
"--",
text,
]
)
logger.info(
"[ui-backend] type_text done len=%d (%.0fms)",
text_len, (time.perf_counter() - t0) * 1000,
)
async def key_press(self, key: str, repeat: int = 1) -> None:
"""xdotool key [--repeat N] <key>。"""
logger.info("[ui-backend] key_press start key=%s repeat=%d", key, repeat)
t0 = time.perf_counter()
async with self._ui_lock:
args = ["xdotool", "key"]
if repeat > 1:
args.extend(["--repeat", str(repeat)])
args.append(key)
await self._run(args)
logger.info(
"[ui-backend] key_press done key=%s repeat=%d (%.0fms)",
key, repeat, (time.perf_counter() - t0) * 1000,
)
async def screenshot(self) -> bytes:
"""scrot 截图输出到临时文件,再读取为 bytes。
注意s6 服务内 /dev/stdout 不是有效设备scrot -o - 会失败
故改用临时文件路径超时 5sTimeoutError/CancelledError kill+wait
"""
import tempfile
async with self._ui_lock:
fd, path = tempfile.mkstemp(suffix=".png", prefix="woc_screenshot_")
os.close(fd)
try:
proc = await asyncio.create_subprocess_exec(
"scrot",
"-o",
path,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
env=self._env(),
)
try:
stdout, stderr = await asyncio.wait_for(
proc.communicate(), timeout=_CMD_TIMEOUT_SEC
)
except (asyncio.TimeoutError, asyncio.CancelledError):
proc.kill()
try:
await asyncio.wait_for(proc.wait(), timeout=5.0)
except asyncio.TimeoutError:
logger.error("[ui-backend] scrot proc.wait() timeout after kill")
raise
except Exception:
if proc.returncode is None:
proc.kill()
try:
await asyncio.wait_for(proc.wait(), timeout=5.0)
except asyncio.TimeoutError:
logger.error(
"[ui-backend] scrot proc.wait() timeout after kill"
)
raise
if proc.returncode != 0:
err = stderr.decode(errors="ignore").strip()[:200]
logger.error("[ui-backend] screenshot failed: rc=%s err=%s", proc.returncode, err)
return b""
with open(path, "rb") as f:
data = f.read()
logger.info(
"[ui-backend] screenshot: scrot rc=0 file=%s bytes=%d",
path, len(data),
)
return data
finally:
try:
os.unlink(path)
except OSError:
pass
async def _find_main_window_id(
self, window_title: str, min_area: int = 10000
) -> int:
"""查找窗口标题匹配者中面积最大的主窗口,过滤隐藏/加载小窗。
微信启动过程中会出现多个窗口隐藏加载窗托盘提示窗等
仅按标题搜索并取第一个常常会拿到 1×1 的无效窗口本方法
1. 搜索所有标题匹配窗口
2. 逐个获取几何并计算面积
3. 过滤掉面积 < min_area 的候选
4. 返回面积最大者
5s TTL 缓存cache_hit 路径上 activate_window + get_window_geometry
+ is_window_active 会重复调用本方法缓存避免 3-5 xdotool search
Args:
window_title: 窗口标题 "微信"
min_area: 最小有效窗口面积默认 10000 像素
Returns:
主窗口 ID
Raises:
RuntimeError: 未找到有效窗口
"""
# 缓存命中检查
cached = self._main_window_id_cache.get(window_title)
if cached is not None:
wid, expire_at = cached
if time.monotonic() < expire_at:
return wid
self._main_window_id_cache.pop(window_title, None)
rc, stdout, _ = await self._run(
["xdotool", "search", "--name", window_title]
)
if rc != 0:
raise RuntimeError(
f"WINDOW_NOT_FOUND: search failed for title={window_title!r}"
)
text = stdout.decode(errors="ignore").strip()
if not text:
raise RuntimeError(
f"WINDOW_NOT_FOUND: no window matches title={window_title!r}"
)
candidates: list[tuple[int, int]] = []
for line in text.splitlines():
wid_str = line.strip()
if not wid_str:
continue
try:
wid = int(wid_str)
except ValueError:
continue
rc, gstdout, _ = await self._run(
["xdotool", "getwindowgeometry", "--shell", str(wid)]
)
if rc != 0:
continue
geom: dict[str, str] = {}
for gline in gstdout.decode(errors="ignore").splitlines():
if "=" in gline:
k, v = gline.split("=", 1)
geom[k.strip()] = v.strip()
try:
w = int(geom.get("WIDTH", 0))
h = int(geom.get("HEIGHT", 0))
area = w * h
if area >= min_area and w >= 200 and h >= 200:
candidates.append((area, wid))
except ValueError:
continue
if not candidates:
raise RuntimeError(
f"WINDOW_NOT_FOUND: no valid window for title={window_title!r} "
f"(candidates filtered by min_area={min_area})"
)
candidates.sort(key=lambda x: x[0], reverse=True)
main_id = candidates[0][1]
# 写入缓存
self._main_window_id_cache[window_title] = (
main_id, time.monotonic() + self._main_window_id_cache_ttl
)
return main_id
async def get_window_geometry(self, window_title: str) -> WindowGeometry:
"""按标题查窗口并返回几何。容忍多个匹配窗口,取面积最大主窗口。
Raises:
RuntimeError: 窗口未找到TODO: P3 时换 FlowError(code=WINDOW_NOT_FOUND)
"""
t0 = time.perf_counter()
logger.info("[ui-backend] get_window_geometry start title=%r", window_title)
try:
window_id = await self._find_main_window_id(window_title)
except RuntimeError as exc:
logger.warning(
"[ui-backend] get_window_geometry: %s (%.0fms)",
exc, (time.perf_counter() - t0) * 1000,
)
raise
rc, stdout, _ = await self._run(
["xdotool", "getwindowgeometry", "--shell", str(window_id)]
)
if rc != 0:
logger.warning(
"[ui-backend] get_window_geometry: getwindowgeometry failed id=%d",
window_id,
)
raise RuntimeError(
f"WINDOW_NOT_FOUND: getwindowgeometry failed for id={window_id}"
)
geom: dict[str, str] = {}
for line in stdout.decode(errors="ignore").splitlines():
if "=" in line:
k, v = line.split("=", 1)
geom[k.strip()] = v.strip()
try:
result = WindowGeometry(
x=int(geom.get("X", 0)),
y=int(geom.get("Y", 0)),
width=int(geom.get("WIDTH", 0)),
height=int(geom.get("HEIGHT", 0)),
)
logger.info(
"[ui-backend] get_window_geometry OK id=%d geom=%dx%d@(%d,%d) (%.0fms)",
window_id, result.width, result.height, result.x, result.y,
(time.perf_counter() - t0) * 1000,
)
return result
except ValueError:
logger.warning(
"[ui-backend] get_window_geometry: invalid geom dict=%s id=%d",
geom, window_id,
)
raise RuntimeError(
f"WINDOW_NOT_FOUND: invalid geometry for id={window_id}"
)
async def activate_window(self, window_title: str) -> None:
"""激活指定标题的窗口。
Raises:
RuntimeError: 窗口未找到TODO: P3 时换 FlowError(code=WINDOW_NOT_FOUND)
"""
t0 = time.perf_counter()
logger.info("[ui-backend] activate_window start title=%r", window_title)
try:
window_id = await self._find_main_window_id(window_title)
except RuntimeError as exc:
logger.warning(
"[ui-backend] activate_window: %s (%.0fms)",
exc, (time.perf_counter() - t0) * 1000,
)
raise
# 使用非阻塞 windowactivate避免 --sync 在 VNC 无人操作时死等。
# 之后轮询 getactivewindow 确认焦点是否真正落到目标窗口。
rc, _, stderr = await self._run(
["xdotool", "windowactivate", str(window_id)]
)
if rc != 0:
err_str = stderr.decode(errors="ignore").strip()[:200] if stderr else ""
logger.warning(
"[ui-backend] activate_window: windowactivate failed id=%d rc=%d err=%s",
window_id, rc, err_str,
)
raise RuntimeError(
f"WINDOW_ACTIVATE_FAILED: windowactivate failed for id={window_id} rc={rc} err={err_str}"
)
deadline = time.monotonic() + 2.0
activated = False
while time.monotonic() < deadline:
rc, stdout, _ = await self._run(["xdotool", "getactivewindow"])
if rc == 0:
active_text = stdout.decode(errors="ignore").strip()
try:
if int(active_text) == window_id:
activated = True
break
except ValueError:
pass
await asyncio.sleep(0.1)
if activated:
logger.info(
"[ui-backend] activate_window OK id=%d title=%r (%.0fms)",
window_id, window_title, (time.perf_counter() - t0) * 1000,
)
else:
logger.warning(
"[ui-backend] activate_window: focus verify timeout id=%d title=%r (%.0fms), continue",
window_id, window_title, (time.perf_counter() - t0) * 1000,
)
async def is_window_active(self, window_title: str) -> bool:
"""校验指定标题的窗口是否是当前活动窗口。
用于纯坐标模式下 _verify_session_still_open 校验窗口焦点
activate_window 调用后焦点校验可能超时只记 warning raise
此方法提供二次校验若窗口已失焦则 cache 失效走完整流程
Returns:
True 表示窗口存在且是活动窗口False 表示窗口不存在或非活动窗口
"""
try:
rc, stdout, _ = await self._run(["xdotool", "getactivewindow"])
if rc != 0:
return False
active_text = stdout.decode(errors="ignore").strip()
try:
active_id = int(active_text)
except ValueError:
return False
main_id = await self._find_main_window_id(window_title)
return active_id == main_id
except Exception as exc:
logger.warning(
"[ui-backend] is_window_active exception title=%r: %s",
window_title, exc,
)
return False
# ------------------------------------------------------------------
# 调试截图异步队列(生产环境关闭,调试模式后台写盘)
# ------------------------------------------------------------------
async def _debug_screenshot_async(self, step: str, trace_id: str = "") -> None:
"""入队调试截图请求,由后台 worker 串行写盘,不阻塞关键路径。
生产环境WOC_UI_DEBUG_SHOTS=false直接 return
"""
if not self._debug_shots_enabled:
return
try:
self._debug_queue.put_nowait((step, trace_id, time.time()))
except asyncio.QueueFull:
logger.warning("[ui-backend] debug queue full, drop shot step=%s", step)
return
if self._debug_task is None or self._debug_task.done():
self._debug_task = asyncio.create_task(self._debug_writer())
async def _debug_writer(self) -> None:
"""后台 worker从队列取 (step, trace_id, ts) 写盘到 /tmp/woc_debug/。
文件名格式{trace_id}_{step}_{ts_ms}.png
异常时记 warning 不抛避免拖垮整个后端
"""
while True:
try:
step, trace_id, ts = await self._debug_queue.get()
except asyncio.CancelledError:
return
try:
os.makedirs("/tmp/woc_debug", exist_ok=True)
path = f"/tmp/woc_debug/{trace_id}_{step}_{int(ts * 1000)}.png"
await self._run(["scrot", path])
except Exception as exc:
logger.warning(
"[ui-backend] debug writer failed step=%s: %s", step, exc
)
@property
def capabilities(self) -> set[str]:
"""返回后端支持的能力集合。"""
return {"click", "type", "key", "screenshot", "window"}