WechatOnCloud/bridge/woc_bridge/ui/flows/send_text.py
Kris c45282f094 feat: 新增好友自动通过、UI自动化能力与多分辨率适配
- 新增登录状态守卫后台任务
- 新增好友申请自动通过规则引擎
- 新增多分辨率UI配置与模板资源
- 新增消息拉取复合游标支持
- 优化发送队列与UI自动化逻辑
- 新增批量发送日志与错误处理
- 优化Docker镜像构建与ptrace初始化
- 新增联系人名称缓存预热
2026-07-17 18:10:31 +08:00

622 lines
26 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""L4 SendTextFlow发送文本消息状态机。
8 个 transitionactivate → click_search_box → type_query → open_session →
focus_input → type_content → send → cleanup。
会话缓存命中时跳过前 4 步,仅做 _verify_session_still_open。
异常时调 _reset_to_idle 恢复 UI 状态。
"""
from __future__ import annotations
import asyncio
import logging
import random
import time
from typing import Awaitable, Callable, Optional
from woc_bridge.ui.backends.base import WindowGeometry
from woc_bridge.ui.errors import FlowError, PermanentError
from woc_bridge.ui.flow import (
Flow,
FlowContext,
FlowResult,
FlowState,
_FLOW_TIMEOUT_SEC,
)
logger = logging.getLogger("woc-bridge")
# 微信窗口标题
_WECHAT_WINDOW_TITLE = "微信"
# DB 校验参数
_DB_VERIFY_TIMEOUT_SEC = 3.0
_DB_VERIFY_INTERVAL_SEC = 0.2
class SendTextFlow(Flow):
"""发送文本消息 Flow。"""
def __init__(self, actions, db_reader=None, session_cache=None) -> None:
super().__init__(actions, db_reader)
self.session_cache = session_cache
# 基线消息的 local_id与 before_msg_idcreate_time配合用于 DB 游标查询
self._before_msg_local_id: int = 0
async def run(self, ctx: FlowContext) -> FlowResult:
"""执行发送文本流程。"""
started_at = time.monotonic()
# 后台 task获取 before_msg_id与 _activate 并行)
before_msg_task: Optional[asyncio.Task] = None
logger.info(
"[flow:send_text] START to_wxid=%s display_name=%s content_len=%d client_request_id=%s",
ctx.to_wxid, ctx.display_name, len(ctx.content), ctx.client_request_id,
)
try:
async with asyncio.timeout(_FLOW_TIMEOUT_SEC):
# 检查会话缓存(用 to_wxid 做 keydisplay_name 非唯一)
cache_hit = False
if self.session_cache is not None:
cache_hit = self.session_cache.get(ctx.to_wxid)
if cache_hit:
logger.info(
"[flow:send_text] session cache hit for %s",
ctx.to_wxid,
)
if cache_hit:
# 跳过 click_search_box ~ open_session但仍需激活窗口
# 30s 内用户可能切到其他应用,微信失去焦点)
t0 = time.perf_counter()
before_msg_task = asyncio.create_task(
self._get_latest_create_time(ctx.to_wxid)
)
# 仅激活窗口 + 获取几何,跳过 Esc 清场(会话已打开)
await self.actions.backend.activate_window(_WECHAT_WINDOW_TITLE)
ctx.win_geom = self._ensure_valid_geometry(
await self.actions.backend.get_window_geometry(
_WECHAT_WINDOW_TITLE
)
)
await self._verify_session_still_open(ctx)
ctx.current_state = FlowState.SESSION_OPENED
logger.info(
"[flow:send_text] cache_hit branch done geom=%dx%d (%.0fms)",
ctx.win_geom.width, ctx.win_geom.height,
(time.perf_counter() - t0) * 1000,
)
else:
# 完整流程
await self._activate(ctx)
# 并行启动 before_msg_id 获取
before_msg_task = asyncio.create_task(
self._get_latest_create_time(ctx.to_wxid)
)
await self._click_search_box(ctx)
await self._type_query(ctx)
await self._open_session(ctx)
await self._focus_input(ctx)
await self._type_content(ctx)
# 等待 before_msg_task 完成(如果已启动)
if before_msg_task is not None:
ct, lid = await before_msg_task
ctx.before_msg_id = ct
self._before_msg_local_id = lid
logger.info(
"[flow:send_text] before_msg baseline ct=%d local_id=%d",
ct, lid,
)
else:
logger.warning(
"[flow:send_text] before_msg_task is None, baseline unavailable"
)
await self._send(ctx)
await self._cleanup(ctx)
# 成功:更新会话缓存(用 to_wxid 做 key
if self.session_cache is not None and ctx.verified:
self.session_cache.set(ctx.to_wxid)
logger.info(
"[flow:send_text] DONE success=%s verified=%s local_id=%s state=%s (%.0fms)",
True, ctx.verified, ctx.local_send_id, ctx.current_state,
(time.monotonic() - started_at) * 1000,
)
return FlowResult(
success=True,
local_id=ctx.local_send_id,
verified=ctx.verified,
state=ctx.current_state,
image_confidence=ctx.image_confidence,
duration_ms=(time.monotonic() - started_at) * 1000,
)
except (FlowError, asyncio.TimeoutError) as exc:
logger.warning(
"[flow:send_text] failed at state=%s: %s: %s",
ctx.current_state, type(exc).__name__, exc,
)
# 尝试恢复
try:
await self._reset_to_idle(ctx.win_geom)
except Exception as reset_exc:
logger.warning(
"[flow:send_text] reset_to_idle failed: %s", reset_exc
)
# 失败时仅失效当前联系人会话缓存
if self.session_cache is not None:
self.session_cache.invalidate(ctx.to_wxid)
error_code = "SEND_FAILED"
error_msg = str(exc)
if isinstance(exc, FlowError) and exc.code:
error_code = exc.code
elif isinstance(exc, asyncio.TimeoutError):
error_code = "SEND_TIMEOUT"
return FlowResult(
success=False,
local_id=ctx.local_send_id,
verified=False,
error=error_msg,
error_code=error_code,
state=FlowState.FAILED,
duration_ms=(time.monotonic() - started_at) * 1000,
)
except Exception as exc:
logger.exception("[flow:send_text] unexpected exception")
try:
await self._reset_to_idle(ctx.win_geom)
except Exception:
pass
if self.session_cache is not None:
self.session_cache.invalidate(ctx.to_wxid)
return FlowResult(
success=False,
error=str(exc),
error_code="BRIDGE_INTERNAL_ERROR",
state=FlowState.FAILED,
duration_ms=(time.monotonic() - started_at) * 1000,
)
finally:
# 兜底:确保 before_msg_task 被取消CancelledError 路径也走这里)
# Python 3.8+ 中 CancelledError 继承自 BaseException
# except Exception 无法捕获finally 块保证清理执行
if before_msg_task is not None and not before_msg_task.done():
before_msg_task.cancel()
try:
await before_msg_task
except (asyncio.CancelledError, Exception):
pass
# ------------------------------------------------------------------
# Transitions
# ------------------------------------------------------------------
def _ensure_valid_geometry(self, geom: Optional[WindowGeometry]) -> WindowGeometry:
"""校验窗口几何有效,防止拿到 1×1 隐藏窗后继续误点 (0,0)。
返回校验后的 WindowGeometry方便调用方获得窄化类型。
"""
if geom is None:
raise PermanentError(
"WINDOW_GEOMETRY_INVALID",
"无法获取微信窗口几何,无法执行 UI 操作",
)
if geom.width < 200 or geom.height < 200:
raise PermanentError(
"WINDOW_GEOMETRY_INVALID",
f"微信窗口几何异常 {geom.width}x{geom.height},无法执行 UI 操作",
)
return geom
async def _activate(self, ctx: FlowContext) -> None:
"""激活窗口 + 获取几何 + 重置状态(回到会话列表首页)。"""
await self.actions.backend.activate_window(_WECHAT_WINDOW_TITLE)
ctx.win_geom = self._ensure_valid_geometry(
await self.actions.backend.get_window_geometry(_WECHAT_WINDOW_TITLE)
)
# 重置状态:点击左侧"会话管理"图标,强制回到会话列表首页,
# 清除可能存在的搜索覆盖层、弹窗或非会话页面。
await self.actions.click_element("session_manager_icon", ctx.win_geom)
await self.actions.backend.key_press("Escape")
await asyncio.sleep(0.3)
ctx.current_state = FlowState.WINDOW_ACTIVATED
logger.info(
"[flow:send_text] activated window geom=%dx%d",
ctx.win_geom.width, ctx.win_geom.height,
)
async def _click_search_box(self, ctx: FlowContext) -> None:
"""点击搜索框。"""
t0 = time.perf_counter()
logger.info("[flow:send_text] step=search_box click start")
await self.actions.click_element("search_box", ctx.win_geom)
ctx.current_state = FlowState.SEARCH_BOX_CLICKED
# 点击后短暂等待 UI 响应,高并发场景下调短以提升吞吐
await asyncio.sleep(0.2)
logger.info(
"[flow:send_text] step=search_box click done (%.0fms)",
(time.perf_counter() - t0) * 1000,
)
async def _type_query(self, ctx: FlowContext) -> None:
"""输入搜索查询display_name"""
t0 = time.perf_counter()
logger.info(
"[flow:send_text] step=type_query start display_name=%r",
ctx.display_name,
)
# 清空可能的残留Ctrl+A 全选 + BackSpace 删除,比 BackSpace×30 更可靠
await self.actions.backend.key_press("ctrl+a")
await self.actions.backend.key_press("BackSpace")
await asyncio.sleep(0.2)
await self.actions.backend.type_text(ctx.display_name)
ctx.current_state = FlowState.QUERY_TYPED
# 自适应等待搜索结果渲染P1 阶段无 OpenCV 时 find_element 的 geom 兜底
# 不可信,因此 _check_search_result_appeared 只在 image 命中时返回 True。
appeared = await self._wait_for_condition(
lambda: self._check_search_result_appeared(ctx),
timeout=1.5,
interval=0.3,
)
if not appeared:
logger.warning(
"[flow:send_text] search result not appeared in 1.5s, continue (%.0fms)",
(time.perf_counter() - t0) * 1000,
)
else:
logger.info(
"[flow:send_text] step=type_query done (search result appeared, %.0fms)",
(time.perf_counter() - t0) * 1000,
)
async def _open_session(self, ctx: FlowContext) -> None:
"""点击搜索结果第一项打开会话。"""
t0 = time.perf_counter()
logger.info("[flow:send_text] step=open_session click search_result_first start")
await self.actions.click_element("search_result_first", ctx.win_geom)
ctx.current_state = FlowState.SESSION_OPENED
# 自适应等待 input_box 出现P1 阶段无 OpenCV 时 find_element 的 geom 兜底
# 不可信,因此 _check_input_box_appeared 只在 image 命中时返回 True。
appeared = await self._wait_for_condition(
lambda: self._check_input_box_appeared(ctx),
timeout=1.5,
interval=0.3,
)
if not appeared:
logger.warning(
"[flow:send_text] input_box not appeared in 1.5s, continue (%.0fms)",
(time.perf_counter() - t0) * 1000,
)
else:
logger.info(
"[flow:send_text] step=open_session done (input_box appeared, %.0fms)",
(time.perf_counter() - t0) * 1000,
)
async def _focus_input(self, ctx: FlowContext) -> None:
"""聚焦输入框。"""
t0 = time.perf_counter()
logger.info("[flow:send_text] step=focus_input click input_box start")
await self.actions.click_element("input_box", ctx.win_geom)
ctx.current_state = FlowState.INPUT_FOCUSED
await asyncio.sleep(0.1)
logger.info(
"[flow:send_text] step=focus_input done (%.0fms)",
(time.perf_counter() - t0) * 1000,
)
async def _type_content(self, ctx: FlowContext) -> None:
"""输入消息内容。"""
t0 = time.perf_counter()
# 清理开头/结尾的空白和换行,避免输入框视觉为空、发送按钮灰色
cleaned_content = ctx.content.strip()
if cleaned_content != ctx.content:
logger.info(
"[flow:send_text] step=type_content stripped leading/trailing whitespace: "
"original_len=%d cleaned_len=%d",
len(ctx.content), len(cleaned_content),
)
ctx.content = cleaned_content
logger.info(
"[flow:send_text] step=type_content start content_len=%d",
len(ctx.content),
)
await self.actions.backend.type_text(ctx.content)
ctx.current_state = FlowState.TEXT_TYPED
logger.info(
"[flow:send_text] step=type_content done (%.0fms)",
(time.perf_counter() - t0) * 1000,
)
async def _send(self, ctx: FlowContext) -> None:
"""点击发送按钮 + 校验。"""
t0 = time.perf_counter()
logger.info("[flow:send_text] step=send click send_button start")
await self.actions.click_element("send_button", ctx.win_geom)
ctx.current_state = FlowState.SENT
ctx.local_send_id = f"local_{int(time.time())}_{random.randint(1000,9999)}"
ctx.verified = await self._verify_sent(ctx)
logger.info(
"[flow:send_text] step=send done local_id=%s verified=%s (%.0fms)",
ctx.local_send_id, ctx.verified,
(time.perf_counter() - t0) * 1000,
)
async def _cleanup(self, ctx: FlowContext) -> None:
"""清理Esc。"""
t0 = time.perf_counter()
logger.info("[flow:send_text] step=cleanup Esc start")
await self.actions.backend.key_press("Escape")
await asyncio.sleep(0.2)
ctx.current_state = FlowState.CLEANED_UP
logger.info(
"[flow:send_text] step=cleanup done (%.0fms)",
(time.perf_counter() - t0) * 1000,
)
# ------------------------------------------------------------------
# 校验
# ------------------------------------------------------------------
async def _verify_sent(self, ctx: FlowContext) -> bool:
"""发送结果校验DB 校验优先,失败降级截图校验。"""
t0 = time.perf_counter()
if self.db_reader is None:
logger.warning(
"[flow:send_text] no db_reader, fall back to screenshot verify"
)
result = await self._verify_by_screenshot(ctx)
logger.info(
"[flow:send_text] _verify_sent done (no db_reader) result=%s (%.0fms)",
result, (time.perf_counter() - t0) * 1000,
)
return result
# before_msg_id=0 表示无法建立基线DB 不可用或会话无历史消息),
# 直接降级截图校验,避免 cursor=0 查询返回所有历史消息导致假阳性
if ctx.before_msg_id <= 0:
logger.warning(
"[flow:send_text] before_msg_id=%d, skip DB verify, fall back to screenshot",
ctx.before_msg_id,
)
result = await self._verify_by_screenshot(ctx)
logger.info(
"[flow:send_text] _verify_sent done (no baseline) result=%s (%.0fms)",
result, (time.perf_counter() - t0) * 1000,
)
return result
deadline = time.monotonic() + _DB_VERIFY_TIMEOUT_SEC
attempts = 0
while time.monotonic() < deadline:
attempts += 1
try:
result = await asyncio.to_thread(
self.db_reader.get_messages_by_session,
ctx.to_wxid,
cursor=ctx.before_msg_id,
cursor_local_id=self._before_msg_local_id,
limit=3,
direction="after",
is_sender=None,
)
messages = result.get("messages", [])
if messages:
# 防串号:校验是否有消息内容匹配发送的 content
# 注意get_messages_by_session 按 talker 分表查询talker 字段恒等于 to_wxid
# 无法通过 talker 字段检测串号。改为内容匹配校验。
for msg in messages:
msg_content = msg.get("content", "")
# 微信消息内容可能包含发送者前缀(群消息),用 in 匹配
if ctx.content and ctx.content in msg_content:
logger.info(
"[flow:send_text] DB verify OK (attempt=%d, content matched, is_sender=%s, %d new msgs)",
attempts, msg.get("is_sender"), len(messages),
)
return True
# 有新消息但内容不匹配 → 可能串号或消息未到达
logger.warning(
"[flow:send_text] DB verify: %d new msgs but content not matched (attempt=%d, contents=%s)",
len(messages), attempts,
[m.get("content", "")[:50] for m in messages],
)
# 继续轮询,可能是 DB 写入延迟
except Exception as exc:
logger.debug(
"[flow:send_text] DB verify attempt=%d exception: %s",
attempts, exc,
)
await asyncio.sleep(_DB_VERIFY_INTERVAL_SEC)
logger.warning(
"[flow:send_text] DB verify timeout after %ds/%d attempts, fall back to screenshot",
_DB_VERIFY_TIMEOUT_SEC, attempts,
)
result = await self._verify_by_screenshot(ctx)
logger.info(
"[flow:send_text] _verify_sent done (DB timeout) result=%s (%.0fms)",
result, (time.perf_counter() - t0) * 1000,
)
return result
async def _verify_by_screenshot(self, ctx: FlowContext) -> bool:
"""截图校验:轮询等待 input_box_empty 模板出现。
策略说明:
- OpenCV image 命中 → 可靠校验通过,返回 True
- geom 兜底命中 → 不可靠,继续轮询等待 image 命中;超时后返回 False
- 超时/异常 → 返回 False
geom 兜底永远返回坐标,无法证明输入框已清空。以往乐观返回 True 会掩盖
真实发送失败,现改为只有 image 命中才认为校验通过。
"""
if ctx.win_geom is None:
logger.warning("[flow:send_text] screenshot verify skipped: win_geom is None")
return False
t0 = time.perf_counter()
deadline = time.monotonic() + 3.0
attempts = 0
last_strategy: Optional[str] = None
last_conf: float = 0.0
while time.monotonic() < deadline:
attempts += 1
try:
elem = await self.actions.find_element(
"input_box_empty", ctx.win_geom
)
if elem is None:
continue
if elem.strategy == "image":
logger.info(
"[flow:send_text] screenshot verify OK (image matched, conf=%.3f, attempts=%d, %.0fms)",
elem.confidence, attempts, (time.perf_counter() - t0) * 1000,
)
return True
# geom 兜底:记录策略,继续轮询看 image 是否能匹配
last_strategy = elem.strategy
last_conf = elem.confidence
except Exception as exc:
logger.debug(
"[flow:send_text] screenshot verify attempt=%d exception: %s",
attempts, exc,
)
await asyncio.sleep(0.3)
# 超时:只有 image 命中才算成功geom 兜底不再乐观返回
if last_strategy == "geom":
logger.warning(
"[flow:send_text] screenshot verify FAIL: only geom fallback matched, "
"image not matched in 3s/%d attempts, %.0fms — cannot prove send success",
attempts, (time.perf_counter() - t0) * 1000,
)
return False
logger.warning(
"[flow:send_text] screenshot verify FAIL: input_box_empty not found in 3s "
"(attempts=%d, %.0fms)",
attempts, (time.perf_counter() - t0) * 1000,
)
return False
async def _verify_session_still_open(self, ctx: FlowContext) -> None:
"""会话缓存命中时校验会话仍打开input_box 存在)。"""
t0 = time.perf_counter()
# 获取窗口几何(缓存命中时未走 _activate并校验/窄化类型
ctx.win_geom = self._ensure_valid_geometry(
await self.actions.backend.get_window_geometry(_WECHAT_WINDOW_TITLE)
)
try:
elem = await self.actions.wait_for(
"input_box", ctx.win_geom, timeout=2.0, interval=0.3
)
if elem is None:
logger.warning(
"[flow:send_text] session_still_open FAIL: input_box not found (%.0fms)",
(time.perf_counter() - t0) * 1000,
)
raise PermanentError(
"ELEMENT_NOT_FOUND",
"session cache hit but input_box not found",
)
logger.info(
"[flow:send_text] session_still_open OK (input_box found, strategy=%s, %.0fms)",
elem.strategy, (time.perf_counter() - t0) * 1000,
)
except PermanentError:
raise
except Exception as exc:
logger.warning(
"[flow:send_text] session_still_open exception: %s (%.0fms)",
exc, (time.perf_counter() - t0) * 1000,
)
raise PermanentError(
"ELEMENT_NOT_FOUND",
f"session cache verify failed: {exc}",
)
# ------------------------------------------------------------------
# 自适应轮询
# ------------------------------------------------------------------
async def _wait_for_condition(
self,
condition_fn: Callable[[], Awaitable[bool]],
timeout: float,
interval: float,
) -> bool:
"""自适应轮询condition_fn 返回 True 时立即返回,超时返回 False。"""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
if await condition_fn():
return True
except Exception:
pass
await asyncio.sleep(interval)
return False
async def _check_search_result_appeared(self, ctx: FlowContext) -> bool:
"""检查搜索结果是否真正出现。
P1 阶段find_element 在 image 匹配失败时会回退到 geom 兜底,
而 geom 兜底只是返回 YAML profile 中写死的比例坐标,不能证明
搜索结果已渲染。因此只有 image 命中strategy == "image"
才视为可靠出现。
"""
try:
elem = await self.actions.find_element(
"search_result_first", ctx.win_geom
)
return elem is not None and elem.strategy == "image"
except Exception:
return False
async def _check_input_box_appeared(self, ctx: FlowContext) -> bool:
"""检查输入框是否真正出现。
同 _check_search_result_appeared只有 image 命中才视为可靠。
"""
try:
elem = await self.actions.find_element("input_box", ctx.win_geom)
return elem is not None and elem.strategy == "image"
except Exception:
return False
# ------------------------------------------------------------------
# DB 辅助
# ------------------------------------------------------------------
async def _get_latest_create_time(self, to_wxid: str) -> tuple[int, int]:
"""获取目标会话当前最新消息的 (create_time, local_id)。"""
if self.db_reader is None:
logger.debug("[flow:send_text] get_latest_create_time: db_reader is None")
return (0, 0)
t0 = time.perf_counter()
try:
result = await asyncio.to_thread(
self.db_reader.get_messages_by_session,
to_wxid,
cursor=0,
limit=1,
direction="before",
is_sender=None,
)
messages = result.get("messages", [])
if messages:
ct = int(messages[0].get("create_time", 0))
lid = int(messages[0].get("local_id", 0))
logger.info(
"[flow:send_text] get_latest_create_time OK to_wxid=%s ct=%d local_id=%d (%.0fms)",
to_wxid, ct, lid, (time.perf_counter() - t0) * 1000,
)
return (ct, lid)
logger.info(
"[flow:send_text] get_latest_create_time: no history msgs for to_wxid=%s (%.0fms)",
to_wxid, (time.perf_counter() - t0) * 1000,
)
except Exception as exc:
logger.warning(
"[flow:send_text] get_latest_create_time exception: %s (%.0fms)",
exc, (time.perf_counter() - t0) * 1000,
)
return (0, 0)