319 lines
12 KiB
Python
319 lines
12 KiB
Python
|
"""L1 xdotool/scrot 后端实现：XdotoolBackend。
|
|||
|
|
|
|||
|
|
通过 xdotool 调用 X11 自动化操作（点击 / 输入 / 按键 / 窗口管理），
|
|||
|
|
scrot 截图输出 stdout 不落盘。所有子进程调用经 _run_managed 上下文管理器，
|
|||
|
|
确保 TimeoutError / CancelledError / 其他异常路径下子进程均被 kill+wait 收尸，
|
|||
|
|
无僵尸进程残留。
|
|||
|
|
|
|||
|
|
分层超时常量：
|
|||
|
|
- _CMD_TIMEOUT_SEC=5.0：单条 xdotool/scrot 命令超时
|
|||
|
|
- _STEP_MIN_TIMEOUT_SEC=7.0：单步 UI 操作最小超时
|
|||
|
|
- _FLOW_TIMEOUT_SEC=30.0：完整发送流程超时
|
|||
|
|
- _HTTP_TIMEOUT_SEC=60.0：HTTP 端到端超时
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import asyncio
|
|||
|
|
import logging
|
|||
|
|
import os
|
|||
|
|
import time
|
|||
|
|
from contextlib import asynccontextmanager
|
|||
|
|
from typing import AsyncIterator, Optional
|
|||
|
|
|
|||
|
|
from woc_bridge.ui.backends.base import BackendProtocol, WindowGeometry
|
|||
|
|
|
|||
|
|
logger = logging.getLogger("woc-bridge")
|
|||
|
|
|
|||
|
|
# 分层超时常量（P1 阶段已落地，供 _run_managed / _step / flow / http 使用）
|
|||
|
|
_CMD_TIMEOUT_SEC = 5.0
|
|||
|
|
_STEP_MIN_TIMEOUT_SEC = 7.0
|
|||
|
|
_FLOW_TIMEOUT_SEC = 30.0
|
|||
|
|
_HTTP_TIMEOUT_SEC = 60.0
|
|||
|
|
|
|||
|
|
|
|||
|
|
class XdotoolBackend(BackendProtocol):
|
|||
|
|
"""xdotool + scrot 后端实现。
|
|||
|
|
|
|||
|
|
所有 UI 操作受 self._ui_lock 串行保护，避免并发点击导致焦点错乱。
|
|||
|
|
asyncio.Lock 必须在事件循环中创建，故在 __init__ 中创建而非模块顶层。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(self, display: str = ":1") -> None:
|
|||
|
|
"""初始化后端。
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
display: X server display 地址，如 ":1"
|
|||
|
|
"""
|
|||
|
|
self.display = display
|
|||
|
|
# asyncio.Lock 必须在事件循环中创建，不在模块顶层
|
|||
|
|
self._ui_lock = asyncio.Lock()
|
|||
|
|
self._debug_shots_enabled = (
|
|||
|
|
os.environ.get("WOC_UI_DEBUG_SHOTS", "false").lower() == "true"
|
|||
|
|
)
|
|||
|
|
self._debug_queue: asyncio.Queue = asyncio.Queue()
|
|||
|
|
self._debug_task: Optional[asyncio.Task] = None
|
|||
|
|
|
|||
|
|
# ------------------------------------------------------------------
|
|||
|
|
# 环境与子进程管理
|
|||
|
|
# ------------------------------------------------------------------
|
|||
|
|
def _env(self) -> dict:
|
|||
|
|
"""返回带 DISPLAY 的环境副本。"""
|
|||
|
|
env = dict(os.environ)
|
|||
|
|
env["DISPLAY"] = self.display
|
|||
|
|
return env
|
|||
|
|
|
|||
|
|
@asynccontextmanager
|
|||
|
|
async def _run_managed(
|
|||
|
|
self, args: list[str]
|
|||
|
|
) -> AsyncIterator[tuple[int, bytes, bytes]]:
|
|||
|
|
"""执行子进程的上下文管理器，确保取消/超时/异常都 kill+wait 收尸。
|
|||
|
|
|
|||
|
|
用法：
|
|||
|
|
async with self._run_managed(args) as (rc, stdout, stderr):
|
|||
|
|
...
|
|||
|
|
|
|||
|
|
任何异常路径均保证 proc.kill() + await proc.wait()，无僵尸残留。
|
|||
|
|
"""
|
|||
|
|
proc = await asyncio.create_subprocess_exec(
|
|||
|
|
*args,
|
|||
|
|
stdout=asyncio.subprocess.PIPE,
|
|||
|
|
stderr=asyncio.subprocess.PIPE,
|
|||
|
|
env=self._env(),
|
|||
|
|
)
|
|||
|
|
try:
|
|||
|
|
stdout, stderr = await asyncio.wait_for(
|
|||
|
|
proc.communicate(), timeout=_CMD_TIMEOUT_SEC
|
|||
|
|
)
|
|||
|
|
except (asyncio.TimeoutError, asyncio.CancelledError):
|
|||
|
|
# 显式 kill + wait，再超时仅记日志
|
|||
|
|
proc.kill()
|
|||
|
|
try:
|
|||
|
|
await asyncio.wait_for(proc.wait(), timeout=5.0)
|
|||
|
|
except asyncio.TimeoutError:
|
|||
|
|
logger.error(
|
|||
|
|
"[ui-backend] proc.wait() timeout after kill: %s", args[0]
|
|||
|
|
)
|
|||
|
|
raise
|
|||
|
|
except Exception:
|
|||
|
|
# 兜底：未退出的子进程一律 kill+wait（带超时，避免无限阻塞 _ui_lock）
|
|||
|
|
if proc.returncode is None:
|
|||
|
|
proc.kill()
|
|||
|
|
try:
|
|||
|
|
await asyncio.wait_for(proc.wait(), timeout=5.0)
|
|||
|
|
except asyncio.TimeoutError:
|
|||
|
|
logger.error(
|
|||
|
|
"[ui-backend] proc.wait() timeout after kill: %s",
|
|||
|
|
args[0] if args else "?",
|
|||
|
|
)
|
|||
|
|
raise
|
|||
|
|
yield (proc.returncode, stdout, stderr)
|
|||
|
|
|
|||
|
|
async def _run(self, args: list[str]) -> tuple[int, bytes, bytes]:
|
|||
|
|
"""执行一条命令并返回 (returncode, stdout, stderr)。"""
|
|||
|
|
async with self._run_managed(args) as result:
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
# ------------------------------------------------------------------
|
|||
|
|
# BackendProtocol 实现
|
|||
|
|
# ------------------------------------------------------------------
|
|||
|
|
async def click(self, x: int, y: int) -> None:
|
|||
|
|
"""合并 mousemove --sync + click 1 为单条 xdotool 命令。
|
|||
|
|
|
|||
|
|
关键：单条命令，避免分两次 fork 之间窗口焦点变化导致点击错位。
|
|||
|
|
"""
|
|||
|
|
async with self._ui_lock:
|
|||
|
|
await self._run(
|
|||
|
|
[
|
|||
|
|
"xdotool",
|
|||
|
|
"mousemove",
|
|||
|
|
"--sync",
|
|||
|
|
str(x),
|
|||
|
|
str(y),
|
|||
|
|
"click",
|
|||
|
|
"1",
|
|||
|
|
]
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
async def type_text(self, text: str) -> None:
|
|||
|
|
"""xdotool type -- <text>：原样输入文本，不解析修饰键组合。"""
|
|||
|
|
async with self._ui_lock:
|
|||
|
|
await self._run(["xdotool", "type", "--", text])
|
|||
|
|
|
|||
|
|
async def key_press(self, key: str, repeat: int = 1) -> None:
|
|||
|
|
"""xdotool key [--repeat N] <key>。"""
|
|||
|
|
async with self._ui_lock:
|
|||
|
|
args = ["xdotool", "key"]
|
|||
|
|
if repeat > 1:
|
|||
|
|
args.extend(["--repeat", str(repeat)])
|
|||
|
|
args.append(key)
|
|||
|
|
await self._run(args)
|
|||
|
|
|
|||
|
|
async def screenshot(self) -> bytes:
|
|||
|
|
"""scrot -o - 输出 PNG 到 stdout，不写文件。
|
|||
|
|
|
|||
|
|
超时 5s，TimeoutError/CancelledError 时 kill+wait。
|
|||
|
|
"""
|
|||
|
|
async with self._ui_lock:
|
|||
|
|
# scrot -o - 输出到 stdout
|
|||
|
|
proc = await asyncio.create_subprocess_exec(
|
|||
|
|
"scrot",
|
|||
|
|
"-o",
|
|||
|
|
"-",
|
|||
|
|
stdout=asyncio.subprocess.PIPE,
|
|||
|
|
stderr=asyncio.subprocess.PIPE,
|
|||
|
|
env=self._env(),
|
|||
|
|
)
|
|||
|
|
try:
|
|||
|
|
stdout, _ = await asyncio.wait_for(
|
|||
|
|
proc.communicate(), timeout=_CMD_TIMEOUT_SEC
|
|||
|
|
)
|
|||
|
|
except (asyncio.TimeoutError, asyncio.CancelledError):
|
|||
|
|
proc.kill()
|
|||
|
|
try:
|
|||
|
|
await asyncio.wait_for(proc.wait(), timeout=5.0)
|
|||
|
|
except asyncio.TimeoutError:
|
|||
|
|
logger.error("[ui-backend] scrot proc.wait() timeout after kill")
|
|||
|
|
raise
|
|||
|
|
except Exception:
|
|||
|
|
if proc.returncode is None:
|
|||
|
|
proc.kill()
|
|||
|
|
try:
|
|||
|
|
await asyncio.wait_for(proc.wait(), timeout=5.0)
|
|||
|
|
except asyncio.TimeoutError:
|
|||
|
|
logger.error(
|
|||
|
|
"[ui-backend] scrot proc.wait() timeout after kill"
|
|||
|
|
)
|
|||
|
|
raise
|
|||
|
|
return stdout
|
|||
|
|
|
|||
|
|
async def get_window_geometry(self, window_title: str) -> WindowGeometry:
|
|||
|
|
"""按标题查窗口并返回几何。容忍多个匹配窗口，取第一个。
|
|||
|
|
|
|||
|
|
Raises:
|
|||
|
|
RuntimeError: 窗口未找到（TODO: P3 时换 FlowError(code=WINDOW_NOT_FOUND)）
|
|||
|
|
"""
|
|||
|
|
# 1. 查窗口 id（容忍多匹配，取第一个）
|
|||
|
|
rc, stdout, _ = await self._run(
|
|||
|
|
["xdotool", "search", "--name", window_title]
|
|||
|
|
)
|
|||
|
|
if rc != 0:
|
|||
|
|
# TODO: P3 时换 FlowError(code="WINDOW_NOT_FOUND", retryable=False)
|
|||
|
|
raise RuntimeError(
|
|||
|
|
f"WINDOW_NOT_FOUND: search failed for title={window_title!r}"
|
|||
|
|
)
|
|||
|
|
text = stdout.decode(errors="ignore").strip()
|
|||
|
|
if not text:
|
|||
|
|
# TODO: P3 时换 FlowError(code="WINDOW_NOT_FOUND", retryable=False)
|
|||
|
|
raise RuntimeError(
|
|||
|
|
f"WINDOW_NOT_FOUND: no window matches title={window_title!r}"
|
|||
|
|
)
|
|||
|
|
first_line = text.splitlines()[0].strip()
|
|||
|
|
try:
|
|||
|
|
window_id = int(first_line)
|
|||
|
|
except ValueError:
|
|||
|
|
# TODO: P3 时换 FlowError(code="WINDOW_NOT_FOUND", retryable=False)
|
|||
|
|
raise RuntimeError(
|
|||
|
|
f"WINDOW_NOT_FOUND: invalid window id={first_line!r} for title={window_title!r}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# 2. 获取几何
|
|||
|
|
rc, stdout, _ = await self._run(
|
|||
|
|
["xdotool", "getwindowgeometry", "--shell", str(window_id)]
|
|||
|
|
)
|
|||
|
|
if rc != 0:
|
|||
|
|
# TODO: P3 时换 FlowError(code="WINDOW_NOT_FOUND", retryable=False)
|
|||
|
|
raise RuntimeError(
|
|||
|
|
f"WINDOW_NOT_FOUND: getwindowgeometry failed for id={window_id}"
|
|||
|
|
)
|
|||
|
|
geom: dict[str, str] = {}
|
|||
|
|
for line in stdout.decode(errors="ignore").splitlines():
|
|||
|
|
if "=" in line:
|
|||
|
|
k, v = line.split("=", 1)
|
|||
|
|
geom[k.strip()] = v.strip()
|
|||
|
|
try:
|
|||
|
|
return WindowGeometry(
|
|||
|
|
x=int(geom.get("X", 0)),
|
|||
|
|
y=int(geom.get("Y", 0)),
|
|||
|
|
width=int(geom.get("WIDTH", 0)),
|
|||
|
|
height=int(geom.get("HEIGHT", 0)),
|
|||
|
|
)
|
|||
|
|
except ValueError:
|
|||
|
|
# TODO: P3 时换 FlowError(code="WINDOW_NOT_FOUND", retryable=False)
|
|||
|
|
raise RuntimeError(
|
|||
|
|
f"WINDOW_NOT_FOUND: invalid geometry for id={window_id}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
async def activate_window(self, window_title: str) -> None:
|
|||
|
|
"""激活指定标题的窗口。
|
|||
|
|
|
|||
|
|
Raises:
|
|||
|
|
RuntimeError: 窗口未找到（TODO: P3 时换 FlowError(code=WINDOW_NOT_FOUND)）
|
|||
|
|
"""
|
|||
|
|
rc, stdout, _ = await self._run(
|
|||
|
|
["xdotool", "search", "--name", window_title]
|
|||
|
|
)
|
|||
|
|
if rc != 0:
|
|||
|
|
# TODO: P3 时换 FlowError(code="WINDOW_NOT_FOUND", retryable=False)
|
|||
|
|
raise RuntimeError(
|
|||
|
|
f"WINDOW_NOT_FOUND: search failed for title={window_title!r}"
|
|||
|
|
)
|
|||
|
|
text = stdout.decode(errors="ignore").strip()
|
|||
|
|
if not text:
|
|||
|
|
# TODO: P3 时换 FlowError(code="WINDOW_NOT_FOUND", retryable=False)
|
|||
|
|
raise RuntimeError(
|
|||
|
|
f"WINDOW_NOT_FOUND: no window matches title={window_title!r}"
|
|||
|
|
)
|
|||
|
|
first_line = text.splitlines()[0].strip()
|
|||
|
|
try:
|
|||
|
|
window_id = int(first_line)
|
|||
|
|
except ValueError:
|
|||
|
|
# TODO: P3 时换 FlowError(code="WINDOW_NOT_FOUND", retryable=False)
|
|||
|
|
raise RuntimeError(
|
|||
|
|
f"WINDOW_NOT_FOUND: invalid window id={first_line!r} for title={window_title!r}"
|
|||
|
|
)
|
|||
|
|
await self._run(["xdotool", "windowactivate", "--sync", str(window_id)])
|
|||
|
|
|
|||
|
|
# ------------------------------------------------------------------
|
|||
|
|
# 调试截图异步队列（生产环境关闭，调试模式后台写盘）
|
|||
|
|
# ------------------------------------------------------------------
|
|||
|
|
async def _debug_screenshot_async(self, step: str, trace_id: str = "") -> None:
|
|||
|
|
"""入队调试截图请求，由后台 worker 串行写盘，不阻塞关键路径。
|
|||
|
|
|
|||
|
|
生产环境（WOC_UI_DEBUG_SHOTS=false）直接 return。
|
|||
|
|
"""
|
|||
|
|
if not self._debug_shots_enabled:
|
|||
|
|
return
|
|||
|
|
try:
|
|||
|
|
self._debug_queue.put_nowait((step, trace_id, time.time()))
|
|||
|
|
except asyncio.QueueFull:
|
|||
|
|
logger.warning("[ui-backend] debug queue full, drop shot step=%s", step)
|
|||
|
|
return
|
|||
|
|
if self._debug_task is None or self._debug_task.done():
|
|||
|
|
self._debug_task = asyncio.create_task(self._debug_writer())
|
|||
|
|
|
|||
|
|
async def _debug_writer(self) -> None:
|
|||
|
|
"""后台 worker：从队列取 (step, trace_id, ts) 写盘到 /tmp/woc_debug/。
|
|||
|
|
|
|||
|
|
文件名格式：{trace_id}_{step}_{ts_ms}.png
|
|||
|
|
异常时记 warning 不抛，避免拖垮整个后端。
|
|||
|
|
"""
|
|||
|
|
while True:
|
|||
|
|
try:
|
|||
|
|
step, trace_id, ts = await self._debug_queue.get()
|
|||
|
|
except asyncio.CancelledError:
|
|||
|
|
return
|
|||
|
|
try:
|
|||
|
|
os.makedirs("/tmp/woc_debug", exist_ok=True)
|
|||
|
|
path = f"/tmp/woc_debug/{trace_id}_{step}_{int(ts * 1000)}.png"
|
|||
|
|
await self._run(["scrot", path])
|
|||
|
|
except Exception as exc:
|
|||
|
|
logger.warning(
|
|||
|
|
"[ui-backend] debug writer failed step=%s: %s", step, exc
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
@property
|
|||
|
|
def capabilities(self) -> set[str]:
|
|||
|
|
"""返回后端支持的能力集合。"""
|
|||
|
|
return {"click", "type", "key", "screenshot", "window"}
|