from __future__ import annotations import asyncio import json import random import re import time from collections import OrderedDict from datetime import datetime, UTC from typing import Any, TYPE_CHECKING from collections.abc import Callable import httpx from yuxi.channels.exceptions import MessageFormatError from yuxi.channels.models import ( ChannelIdentity, ChannelMessage, ChannelType, ChatType, MentionsInfo, ) from yuxi.utils.logging_config import logger from .format import ( parse_graph_update, ensure_ship_tilde, extract_graph_mentions, extract_graph_attachments, ) from .cites import parse_cite if TYPE_CHECKING: from .client import UrbitClient from .threads import ThreadManager from .history import MessageCache _MAX_RECONNECT_DELAY = 30.0 _INITIAL_RECONNECT_DELAY = 1.0 _MAX_RECONNECT_COUNT = 10 _MAX_CONSECUTIVE_ERRORS = 5 _JITTER = 0.1 _ACK_INTERVAL = 20 _ALL_PATTERN = re.compile(r"\b@(all|everyone|channel|here)\b", re.IGNORECASE) class SSEDeduplicator: def __init__(self, max_size: int = 2000): self._seen: OrderedDict[str, float] = OrderedDict() self._max_size = max_size self._lock = asyncio.Lock() async def is_duplicate(self, graph_index: str) -> bool: async with self._lock: now = time.monotonic() if graph_index in self._seen: if now - self._seen[graph_index] < 60: return True self._seen[graph_index] = now self._seen.move_to_end(graph_index) if len(self._seen) > self._max_size: self._seen.popitem(last=False) return False class UrbitSSEManager: """多订阅 SSE 管理器,负责 SSE 连接的创建、ACK 和优雅关闭""" def __init__( self, client: UrbitClient, ship_url: str, on_message: Callable[[ChannelMessage], Any], status_check: Callable[[], bool], reconnect_delay: float = 5.0, dedup: SSEDeduplicator | None = None, bot_ship_name: str = "", reauth_callback: Callable[[], Any] | None = None, thread_manager: ThreadManager | None = None, message_cache: MessageCache | None = None, ): self._client = client self._ship_url = ship_url self._on_message = on_message self._status_check = status_check self._reconnect_delay = reconnect_delay self._dedup = dedup or SSEDeduplicator() self._bot_ship_name = bot_ship_name.lstrip("~") self._subscriptions: dict[str, asyncio.Task | None] = {} self._channel_ids: dict[str, str] = {} self._event_counts: dict[str, int] = {} self._run_lock = asyncio.Lock() self._reauth_callback = reauth_callback self._thread_manager = thread_manager self._message_cache = message_cache self._contacts: dict[str, str] = {} self._on_foreign_update: Callable[[dict[str, Any]], Any] | None = None self._on_groups_ui_update: Callable[[dict[str, Any]], Any] | None = None self._on_settings_update: Callable[[dict[str, Any]], Any] | None = None async def start(self, subscriptions: dict[str, str] | None = None) -> None: """启动 SSE 订阅管理器 subscriptions: {label: sse_path},默认监听 chat-store """ if subscriptions is None: subscriptions = {"chat": "/~/channel/chat-store"} for label, sse_path in subscriptions.items(): channel_id = f"{int(time.time() * 1000)}-{label}" self._channel_ids[label] = channel_id self._event_counts[label] = 0 task = asyncio.create_task(self._run_subscription(label, sse_path, channel_id)) self._subscriptions[label] = task logger.info(f"[Urbit] SSE subscription '{label}' → {sse_path} (channel: {channel_id})") async def stop(self) -> None: for label, task in list(self._subscriptions.items()): if task and not task.done(): task.cancel() try: await task except asyncio.CancelledError: pass await self._unsubscribe_all() for label, channel_id in list(self._channel_ids.items()): try: await self._unsubscribe_channel(channel_id) except Exception as e: logger.warning(f"[Urbit] Failed to unsubscribe channel '{label}': {e}") await self._release_streams() self._subscriptions.clear() self._channel_ids.clear() self._event_counts.clear() logger.info("[Urbit] All SSE subscriptions stopped") async def _unsubscribe_all(self) -> None: task_count = 0 for label, task in list(self._subscriptions.items()): if task and not task.done(): task.cancel() task_count += 1 if task_count > 0: await asyncio.sleep(0.1) logger.debug(f"[Urbit] Unsubscribed all {task_count} SSE tasks") async def _release_streams(self) -> None: try: await self._client.http.aclose() except Exception: pass logger.debug("[Urbit] SSE streams released") async def _unsubscribe_channel(self, channel_id: str) -> None: try: r = await self._client.delete(f"/~/channel/{channel_id}") if r.status_code not in (200, 204): logger.debug(f"[Urbit] Channel DELETE {channel_id} → {r.status_code}") except Exception as e: logger.debug(f"[Urbit] Channel DELETE {channel_id} failed: {e}") async def _send_ack(self, channel_id: str) -> None: try: await self._client.put( f"/~/channel/{channel_id}", json={"action": "ack"}, timeout=5.0, ) except Exception: pass async def _run_subscription(self, label: str, sse_path: str, channel_id: str) -> None: delay = _INITIAL_RECONNECT_DELAY reconnect_count = 0 consecutive_errors = 0 while self._status_check(): try: response = await self._client.stream_get(sse_path) response.raise_for_status() logger.info(f"[Urbit] SSE '{label}' connected to {self._ship_url}{sse_path}") delay = _INITIAL_RECONNECT_DELAY reconnect_count = 0 consecutive_errors = 0 self._event_counts[label] = 0 async for line in response.aiter_lines(): if not self._status_check(): break if not line or not line.startswith("data:"): continue data_str = line[5:].strip() if not data_str: continue try: data = json.loads(data_str) except json.JSONDecodeError: continue label_handlers = { "chat": lambda d: self._process_chat_event(d, channel_id, label), "channels": lambda d: self._process_channels_event(d, channel_id, label), "contacts": lambda d: self._process_contacts_event(d, label), "foreigns": lambda d: self._process_groups_event(d, label), "settings": lambda d: self._process_settings_event(d, label), "groups-ui": lambda d: self._process_groups_ui_event(d, label), } handler = label_handlers.get(label) if handler: await handler(data) else: await self._process_generic_event(data, label) except httpx.RequestError as e: consecutive_errors = 0 logger.warning(f"[Urbit] SSE '{label}' connection lost: {e}. Reconnecting in {delay:.1f}s...") except Exception as e: consecutive_errors += 1 logger.error( f"[Urbit] SSE '{label}' unexpected error (#{consecutive_errors}): {e}" ) if consecutive_errors >= _MAX_CONSECUTIVE_ERRORS: logger.critical( f"[Urbit] SSE '{label}' aborted after {consecutive_errors} consecutive errors" ) break reconnect_count += 1 if reconnect_count > _MAX_RECONNECT_COUNT: logger.error( f"[Urbit] SSE '{label}' exceeded max reconnect count ({_MAX_RECONNECT_COUNT}), pausing 10s..." ) await asyncio.sleep(10.0) reconnect_count = 0 delay = _INITIAL_RECONNECT_DELAY if self._reauth_callback: try: await self._reauth_callback() logger.info(f"[Urbit] SSE '{label}' re-authenticated before reconnect") except Exception as e: logger.warning(f"[Urbit] SSE '{label}' re-auth failed: {e}") jitter_ms = delay * _JITTER * random.random() await asyncio.sleep(delay + jitter_ms) delay = min(delay * 2, _MAX_RECONNECT_DELAY) async def _process_chat_event(self, data: dict[str, Any], channel_id: str, label: str) -> None: graph_update = data.get("graph-update") if not graph_update: return edit = graph_update.get("edit") if edit and isinstance(edit, dict): self._handle_edit_event(edit, channel_id, label) return index = str(graph_update.get("index", "")) if index and await self._dedup.is_duplicate(index): return self._event_counts[label] = self._event_counts.get(label, 0) + 1 if self._event_counts[label] % _ACK_INTERVAL == 0: asyncio.create_task(self._send_ack(channel_id)) try: msg = _graph_update_to_channel_message(data, self._client.ship_name) if not msg.content: return if _is_self_message(msg, self._bot_ship_name): return if msg.attachments: attachment_hints = [] for att in msg.attachments: url = att.get("url", "") content_type = att.get("mime_type", "application/octet-stream") attachment_hints.append(f"[media attached: {url} ({content_type})]") if attachment_hints: hint_text = " ".join(attachment_hints) msg.content = f"{hint_text}\n{msg.content}" if self._thread_manager and self._thread_manager.is_thread_reply(data): parent_id = self._thread_manager.get_reply_parent_id(data) if parent_id: msg.metadata["thread_parent_id"] = parent_id thread_key = f"{msg.identity.channel_chat_id}:{parent_id}" if not self._thread_manager.has_participated(thread_key): mentions = _detect_bot_mentions(msg, self._bot_ship_name) if not mentions: logger.debug( f"[Urbit] Thread reply without mention in {msg.identity.channel_chat_id}, " f"skipping (not participated)" ) return mentions = _detect_bot_mentions(msg, self._bot_ship_name) if mentions: msg.mentions = mentions if self._thread_manager: parent_id = msg.metadata.get("thread_parent_id") if parent_id: thread_key = f"{msg.identity.channel_chat_id}:{parent_id}" self._thread_manager.add_participated(thread_key) asyncio.create_task(self._thread_manager.persist_participated()) cites_info = _extract_cites_from_message(msg) if cites_info: msg.metadata["cites"] = cites_info await self._on_message(msg) if self._message_cache and msg.metadata.get("msg_id"): await self._message_cache.cache_message( msg.metadata["msg_id"], msg.identity.channel_chat_id, msg.content, msg.metadata.get("urbit_ship", ""), ) except MessageFormatError: logger.warning(f"[Urbit] SSE unparseable event: {json.dumps(data)[:200]}") except Exception as e: logger.error(f"[Urbit] SSE event processing error: {e}") def _handle_edit_event(self, edit: dict[str, Any], channel_id: str, label: str) -> None: index = edit.get("index", "") content_blocks = edit.get("content", []) extracted = self._extract_edit_content(content_blocks) if extracted: logger.info(f"[Urbit] SSE edit event at index={index}: content updated ({len(extracted)} chars)") @staticmethod def _extract_edit_content(contents: list[dict[str, Any]]) -> str: parts: list[str] = [] for item in contents: if isinstance(item, dict) and "text" in item: parts.append(item["text"]) return "".join(parts) async def _process_generic_event(self, data: dict[str, Any], label: str) -> None: logger.debug(f"[Urbit] SSE '{label}' event: {json.dumps(data)[:200]}") async def _process_channels_event(self, data: dict[str, Any], channel_id: str, label: str) -> None: add = data.get("add") if not add or not isinstance(add, dict): return nest = add.get("nest", "") post = add.get("post", {}) if not post: return index = post.get("index", "") if index and await self._dedup.is_duplicate(f"chan:{nest}:{index}"): return self._event_counts[label] = self._event_counts.get(label, 0) + 1 if self._event_counts[label] % _ACK_INTERVAL == 0: asyncio.create_task(self._send_ack(channel_id)) graph_data = { "graph-update": { "resource": {"type": "chat", "path": nest}, "ship": post.get("author", ""), "index": index, "time": post.get("time-sent"), "additions": {index: {"post": post}}, } } try: msg = _graph_update_to_channel_message(graph_data, self._client.ship_name) if not msg.content: return if _is_self_message(msg, self._bot_ship_name): return if msg.attachments: attachment_hints = [] for att in msg.attachments: url = att.get("url", "") content_type = att.get("mime_type", "application/octet-stream") attachment_hints.append(f"[media attached: {url} ({content_type})]") if attachment_hints: hint_text = " ".join(attachment_hints) msg.content = f"{hint_text}\n{msg.content}" if self._thread_manager and self._thread_manager.is_thread_reply(graph_data): parent_id = self._thread_manager.get_reply_parent_id(graph_data) if parent_id: msg.metadata["thread_parent_id"] = parent_id thread_key = f"{msg.identity.channel_chat_id}:{parent_id}" if not self._thread_manager.has_participated(thread_key): mentions = _detect_bot_mentions(msg, self._bot_ship_name) if not mentions: logger.debug( f"[Urbit] Channels thread reply without mention in {msg.identity.channel_chat_id}, " f"skipping (not participated)" ) return mentions = _detect_bot_mentions(msg, self._bot_ship_name) if mentions: msg.mentions = mentions if self._thread_manager: parent_id = msg.metadata.get("thread_parent_id") if parent_id: thread_key = f"{msg.identity.channel_chat_id}:{parent_id}" self._thread_manager.add_participated(thread_key) asyncio.create_task(self._thread_manager.persist_participated()) cites_info = _extract_cites_from_message(msg) if cites_info: msg.metadata["cites"] = cites_info await self._on_message(msg) if self._message_cache and msg.metadata.get("msg_id"): await self._message_cache.cache_message( msg.metadata["msg_id"], msg.identity.channel_chat_id, msg.content, msg.metadata.get("urbit_ship", ""), ) except MessageFormatError: logger.warning(f"[Urbit] SSE channels unparseable event: {json.dumps(data)[:200]}") except Exception as e: logger.error(f"[Urbit] SSE channels event processing error: {e}") async def _process_contacts_event(self, data: dict[str, Any], label: str) -> None: con = data.get("con") if not con or not isinstance(con, dict): return who = con.get("who", "") nick = con.get("nick", "") if who: ship = who.lstrip("~") if nick: self._contacts[ship] = nick logger.info(f"[Urbit] Contact update: ~{ship} nick='{nick}'") else: self._contacts.pop(ship, None) logger.info(f"[Urbit] Contact removed: ~{ship}") async def _process_groups_event(self, data: dict[str, Any], label: str) -> None: foreign_update = data.get("foreignUpdate") if not foreign_update or not isinstance(foreign_update, dict): return group = foreign_update.get("group", "") ship = foreign_update.get("ship", "") join = foreign_update.get("join", False) if group and ship: action = "joined" if join else "left" logger.info(f"[Urbit] Group foreign update: ~{ship} {action} {group}") if self._on_foreign_update is not None: await self._on_foreign_update(foreign_update) def set_foreign_update_handler(self, handler: Callable[[dict[str, Any]], Any]) -> None: self._on_foreign_update: Callable[[dict[str, Any]], Any] | None = handler async def _process_groups_ui_event(self, data: dict[str, Any], label: str) -> None: add = data.get("add") if add and isinstance(add, dict): group_data = add.get("group", {}) channels = add.get("channels", []) group_id = group_data.get("id", "") if isinstance(group_data, dict) else "" if group_id: logger.info(f"[Urbit] Groups-UI add: group={group_id}, channels={channels}") if self._on_groups_ui_update is not None: await self._on_groups_ui_update({"action": "add", "group_id": group_id, "channels": channels}) return join = data.get("join") if join and isinstance(join, dict): group_path = join.get("group", "") ship = join.get("ship", "") if group_path: logger.info(f"[Urbit] Groups-UI join: group={group_path}, ship={ship}") if self._on_groups_ui_update is not None: await self._on_groups_ui_update({"action": "join", "group_path": group_path, "ship": ship}) return kick = data.get("kick") if kick and isinstance(kick, dict): group_path = kick.get("group", "") ship = kick.get("ship", "") if group_path: logger.info(f"[Urbit] Groups-UI kick: group={group_path}, ship={ship}") if self._on_groups_ui_update is not None: await self._on_groups_ui_update({"action": "kick", "group_path": group_path, "ship": ship}) def set_groups_ui_update_handler(self, handler: Callable[[dict[str, Any]], Any]) -> None: self._on_groups_ui_update: Callable[[dict[str, Any]], Any] | None = handler @property def contacts(self) -> dict[str, str]: return dict(self._contacts) async def _process_settings_event(self, data: dict[str, Any], label: str) -> None: settings_update = data.get("settings-event") if not settings_update: return try: put_entry = settings_update.get("put-entry") del_entry = settings_update.get("del-entry") if put_entry and isinstance(put_entry, dict): bucket_key = put_entry.get("bucket-key", "") entry_key = put_entry.get("entry-key", "") value = put_entry.get("value") if bucket_key and entry_key: if value is not None: try: parsed = json.loads(value) if isinstance(value, str) else value except json.JSONDecodeError: logger.warning(f"[Urbit] Settings put-entry invalid JSON for {bucket_key}/{entry_key}") parsed = value logger.info(f"[Urbit] Settings hot-reload: put-entry {bucket_key}/{entry_key}") if self._on_settings_update is not None: await self._on_settings_update( { "action": "put", "bucket_key": bucket_key, "entry_key": entry_key, "value": parsed, } ) else: logger.debug(f"[Urbit] Settings put-entry null value for {bucket_key}/{entry_key}") elif del_entry and isinstance(del_entry, dict): bucket_key = del_entry.get("bucket-key", "") entry_key = del_entry.get("entry-key", "") if bucket_key and entry_key: logger.info(f"[Urbit] Settings hot-reload: del-entry {bucket_key}/{entry_key}") if self._on_settings_update is not None: await self._on_settings_update( { "action": "del", "bucket_key": bucket_key, "entry_key": entry_key, } ) else: logger.debug(f"[Urbit] Settings Store event: {json.dumps(data)[:200]}") except Exception as e: logger.warning(f"[Urbit] Settings event processing error: {e}") def set_settings_update_handler(self, handler: Callable[[dict[str, Any]], Any]) -> None: self._on_settings_update: Callable[[dict[str, Any]], Any] | None = handler @property def dedup(self) -> SSEDeduplicator: return self._dedup @property def active_subscriptions(self) -> list[str]: return [label for label, t in self._subscriptions.items() if t and not t.done()] async def run_sse_listener( client: UrbitClient, ship_url: str, on_message: Callable[[ChannelMessage], Any], status_check: Callable[[], bool], reconnect_delay: float = 5.0, dedup: SSEDeduplicator | None = None, ) -> None: manager = UrbitSSEManager( client=client, ship_url=ship_url, on_message=on_message, status_check=status_check, reconnect_delay=reconnect_delay, dedup=dedup, ) await manager.start({"chat": "/~/channel/chat-store"}) for task in manager._subscriptions.values(): if task: await task def _is_self_message(msg: ChannelMessage, bot_ship: str) -> bool: if not bot_ship: return False sender = msg.metadata.get("urbit_ship", "").lower() return sender == bot_ship.lower() def _extract_cites_from_message(msg: ChannelMessage) -> list[dict[str, str]]: additions = msg.metadata.get("raw_additions", {}) cites_list: list[dict[str, str]] = [] for node_data in additions.values(): post = node_data.get("post", {}) contents = post.get("contents", []) for item in contents: if isinstance(item, dict) and "cite" in item: cite_result = parse_cite(item) if cite_result: cites_list.append(cite_result) return cites_list def _detect_bot_mentions(msg: ChannelMessage, bot_ship: str) -> MentionsInfo | None: content = msg.content or "" if _ALL_PATTERN.search(content): return MentionsInfo( mentioned_user_ids=["@all"], is_bot_mentioned=True, ) if not bot_ship: return msg.mentions if not content: return None mentioned = False bot_with_tilde = f"~{bot_ship}" if re.search(rf"\b@{bot_ship}\b", content, re.IGNORECASE): mentioned = True if re.search(rf"\b{bot_with_tilde}\b", content, re.IGNORECASE): mentioned = True if not mentioned and "additions" in str(msg.metadata): additions = msg.metadata.get("raw_additions", {}) mentions_list = extract_graph_mentions(additions) if bot_ship.lower() in (m.lower() for m in mentions_list): mentioned = True if mentioned: return MentionsInfo( mentioned_user_ids=[f"urbit:{bot_ship}"], is_bot_mentioned=True, ) return msg.mentions def _graph_update_to_channel_message( raw: dict[str, Any], ship_name: str, ) -> ChannelMessage: parsed = parse_graph_update(raw) sender_ship = parsed.get("ship", ship_name) resource_path = parsed.get("resource_path", "") resource_type = parsed.get("resource_type", "chat") index = parsed.get("index", "") chat_type_str = "direct" if resource_type == "dm" else "group" chat_type = ChatType.DIRECT if resource_type == "dm" else ChatType.GROUP if resource_type in ("diary", "heap"): chat_type_str = "group" chat_type = ChatType.GROUP identity = ChannelIdentity( channel_id="urbit", channel_type=ChannelType.URBIT, channel_user_id=ensure_ship_tilde(sender_ship), channel_chat_id=resource_path or "unknown", channel_message_id=str(index) if index else None, ) content = parsed.get("content", "") timestamp = None raw_time = parsed.get("time") if raw_time: try: timestamp = datetime.fromtimestamp(float(raw_time), tz=UTC) except (TypeError, ValueError): timestamp = None metadata: dict[str, Any] = { "urbit_resource_type": resource_type, "urbit_ship": sender_ship, "chat_type": chat_type_str, } if "reaction" in parsed: metadata["reaction"] = parsed["reaction"] additions = raw.get("graph-update", {}).get("additions", {}) if additions: metadata["raw_additions"] = additions mentions = extract_graph_mentions(additions) mentions_info = None if mentions: mentions_info = MentionsInfo( mentioned_user_ids=[f"urbit:{m}" for m in mentions], is_bot_mentioned=False, ) attachments = extract_graph_attachments(additions) return ChannelMessage( identity=identity, chat_type=chat_type, content=content.strip(), attachments=attachments, mentions=mentions_info, metadata=metadata, timestamp=timestamp, )