From 49ecd949e8b36e0cdd02b56a7d8524901b87d2d5 Mon Sep 17 00:00:00 2001 From: Kris <2893855659@qq.com> Date: Tue, 12 May 2026 00:50:27 +0800 Subject: [PATCH] =?UTF-8?q?feat(urbit):=20=E5=AE=9E=E7=8E=B0=E5=AE=8C?= =?UTF-8?q?=E6=95=B4=E7=9A=84Urbit=E8=81=8A=E5=A4=A9=E9=80=82=E9=85=8D?= =?UTF-8?q?=E5=99=A8=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增了从错误定义、客户端实现到功能完整的Urbit适配器全套代码，包括认证、消息处理、目录查询、邀请管理、媒体处理、速率限制等核心功能模块 --- .../yuxi/channels/adapters/urbit/__init__.py | 30 + .../yuxi/channels/adapters/urbit/accounts.py | 206 ++++ .../yuxi/channels/adapters/urbit/adapter.py | 913 ++++++++++++++++++ .../yuxi/channels/adapters/urbit/approval.py | 259 +++++ .../yuxi/channels/adapters/urbit/auth.py | 39 + .../channels/adapters/urbit/authorization.py | 59 ++ .../yuxi/channels/adapters/urbit/cites.py | 48 + .../yuxi/channels/adapters/urbit/client.py | 96 ++ .../adapters/urbit/config_ui_hints.py | 398 ++++++++ .../yuxi/channels/adapters/urbit/directory.py | 68 ++ .../yuxi/channels/adapters/urbit/discovery.py | 121 +++ .../yuxi/channels/adapters/urbit/doctor.py | 51 + .../channels/adapters/urbit/dual_plugin.py | 154 +++ .../yuxi/channels/adapters/urbit/errors.py | 19 + .../yuxi/channels/adapters/urbit/format.py | 259 +++++ .../yuxi/channels/adapters/urbit/history.py | 66 ++ .../yuxi/channels/adapters/urbit/invites.py | 70 ++ .../yuxi/channels/adapters/urbit/media.py | 57 ++ .../yuxi/channels/adapters/urbit/monitor.py | 712 ++++++++++++++ .../yuxi/channels/adapters/urbit/poke_api.py | 138 +++ .../yuxi/channels/adapters/urbit/probe.py | 95 ++ .../channels/adapters/urbit/rate_limiter.py | 45 + .../yuxi/channels/adapters/urbit/scry.py | 67 ++ .../yuxi/channels/adapters/urbit/send.py | 411 ++++++++ .../yuxi/channels/adapters/urbit/session.py | 59 ++ .../yuxi/channels/adapters/urbit/settings.py | 177 ++++ .../yuxi/channels/adapters/urbit/setup.py | 208 ++++ .../yuxi/channels/adapters/urbit/story.py | 316 ++++++ .../channels/adapters/urbit/summarizer.py | 76 ++ .../yuxi/channels/adapters/urbit/targets.py | 43 + .../yuxi/channels/adapters/urbit/threads.py | 127 +++ .../yuxi/channels/adapters/urbit/upload.py | 230 +++++ .../yuxi/channels/adapters/urbit/vision.py | 96 ++ 33 files changed, 5713 insertions(+) create mode 100644 backend/package/yuxi/channels/adapters/urbit/__init__.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/accounts.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/adapter.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/approval.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/auth.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/authorization.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/cites.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/client.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/config_ui_hints.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/directory.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/discovery.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/doctor.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/dual_plugin.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/errors.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/format.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/history.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/invites.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/media.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/monitor.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/poke_api.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/probe.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/rate_limiter.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/scry.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/send.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/session.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/settings.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/setup.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/story.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/summarizer.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/targets.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/threads.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/upload.py create mode 100644 backend/package/yuxi/channels/adapters/urbit/vision.py diff --git a/backend/package/yuxi/channels/adapters/urbit/__init__.py b/backend/package/yuxi/channels/adapters/urbit/__init__.py new file mode 100644 index 00000000..0568c39a --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/__init__.py @@ -0,0 +1,30 @@ +from yuxi.channels.adapters.urbit.adapter import UrbitAdapter +from yuxi.channels.adapters.urbit.accounts import AccountManager, MultiAccountConnectionManager +from yuxi.channels.adapters.urbit.invites import InviteManager +from yuxi.channels.adapters.urbit.directory import DirectoryQuery +from yuxi.channels.adapters.urbit.dual_plugin import ( + UrbitSetupPlugin, + UrbitAccountPlugin, + DualPluginRegistry, + get_global_registry, +) +from yuxi.channels.adapters.urbit.poke_api import HttpPokeApiClient, with_http_poke_api, create_http_poke_api +from yuxi.channels.adapters.urbit.config_ui_hints import get_config_ui_hints, get_categories, get_hints_by_category + +__all__ = [ + "UrbitAdapter", + "AccountManager", + "MultiAccountConnectionManager", + "InviteManager", + "DirectoryQuery", + "UrbitSetupPlugin", + "UrbitAccountPlugin", + "DualPluginRegistry", + "get_global_registry", + "HttpPokeApiClient", + "with_http_poke_api", + "create_http_poke_api", + "get_config_ui_hints", + "get_categories", + "get_hints_by_category", +] diff --git a/backend/package/yuxi/channels/adapters/urbit/accounts.py b/backend/package/yuxi/channels/adapters/urbit/accounts.py new file mode 100644 index 00000000..64f25426 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/accounts.py @@ -0,0 +1,206 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Callable +from typing import Any + +from yuxi.utils.logging_config import logger + + +class AccountInfo: + def __init__(self, name: str, ship: str, url: str, code: str, metadata: dict[str, Any] | None = None): + self.name = name + self.ship = ship.lstrip("~") + self.url = url.rstrip("/") + self.code = code + self.metadata = metadata or {} + self.status: str = "disconnected" + self.last_probe_at: float = 0.0 + + @property + def ship_with_tilde(self) -> str: + return f"~{self.ship}" + + def to_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "ship": self.ship_with_tilde, + "url": self.url, + "status": self.status, + "metadata": self.metadata, + } + + +class AccountManager: + def __init__(self, accounts_config: list[dict[str, Any]] | None = None): + self._accounts: dict[str, AccountInfo] = {} + self._on_status_change: list[Callable[[str, str], Any]] = [] + if accounts_config: + self.load_from_config(accounts_config) + + def load_from_config(self, accounts_config: list[dict[str, Any]]) -> None: + self._accounts.clear() + for acct in accounts_config: + name = acct.get("name", acct.get("ship", "default")) + ship = acct.get("ship", "").lstrip("~") + url = acct.get("url", "") + code = acct.get("code", "") + if not ship or not url or not code: + logger.warning(f"[Urbit] Skipping invalid account: {name}") + continue + self._accounts[name] = AccountInfo( + name=name, + ship=ship, + url=url, + code=code, + metadata=acct.get("metadata", {}), + ) + logger.info(f"[Urbit] Loaded {len(self._accounts)} accounts") + + def add_account(self, name: str, ship: str, url: str, code: str, **meta) -> AccountInfo: + info = AccountInfo(name=name, ship=ship, url=url, code=code, metadata=meta) + self._accounts[name] = info + logger.info(f"[Urbit] Account '{name}' added (~{info.ship})") + return info + + def remove_account(self, name: str) -> bool: + if name in self._accounts: + del self._accounts[name] + logger.info(f"[Urbit] Account '{name}' removed") + return True + return False + + def get(self, name: str = "default") -> AccountInfo | None: + if not self._accounts and name == "default": + return None + return self._accounts.get(name) + + def get_all(self) -> dict[str, AccountInfo]: + return dict(self._accounts) + + def list_ships(self) -> list[str]: + return [f"~{a.ship}" for a in self._accounts.values()] + + def set_status(self, name: str, status: str) -> None: + acct = self._accounts.get(name) + if acct: + old = acct.status + acct.status = status + if old != status: + for handler in self._on_status_change: + try: + handler(name, status) + except Exception as e: + logger.warning(f"[Urbit] Status change handler error: {e}") + + def on_status_change(self, handler: Callable[[str, str], Any]) -> None: + self._on_status_change.append(handler) + + @property + def is_empty(self) -> bool: + return len(self._accounts) == 0 + + @property + def count(self) -> int: + return len(self._accounts) + + def has_multi_accounts(self) -> bool: + return len(self._accounts) > 1 + + +class MultiAccountConnectionManager: + def __init__( + self, + account_manager: AccountManager, + connection_factory: Callable[[AccountInfo], Any], + ): + self._account_manager = account_manager + self._connection_factory = connection_factory + self._connections: dict[str, Any] = {} + self._tasks: dict[str, asyncio.Task | None] = {} + self._running = False + + @property + def connections(self) -> dict[str, Any]: + return dict(self._connections) + + @property + def is_running(self) -> bool: + return self._running + + async def start_all(self) -> None: + if self._running: + return + + self._running = True + accounts = self._account_manager.get_all() + + for name, account in accounts.items(): + await self._start_account(name, account) + + logger.info(f"[Urbit] MultiAccountConnectionManager started with {len(self._connections)} connections") + + async def _start_account(self, name: str, account: AccountInfo) -> None: + try: + connection = await self._connection_factory(account) + self._connections[name] = connection + self._account_manager.set_status(name, "connected") + logger.info(f"[Urbit] Account '{name}' (~{account.ship}) connected") + except Exception as e: + self._account_manager.set_status(name, "error") + logger.error(f"[Urbit] Failed to connect account '{name}' (~{account.ship}): {e}") + + async def start_account(self, name: str, account: AccountInfo) -> None: + if name in self._connections: + logger.warning(f"[Urbit] Account '{name}' already connected") + return + await self._start_account(name, account) + + async def stop_account(self, name: str) -> None: + connection = self._connections.pop(name, None) + if connection is None: + return + + if hasattr(connection, "disconnect"): + try: + await connection.disconnect() + except Exception as e: + logger.warning(f"[Urbit] Error disconnecting account '{name}': {e}") + + self._account_manager.set_status(name, "disconnected") + logger.info(f"[Urbit] Account '{name}' disconnected") + + async def stop_all(self) -> None: + if not self._running: + return + + self._running = False + names = list(self._connections.keys()) + for name in names: + await self.stop_account(name) + + self._connections.clear() + logger.info("[Urbit] MultiAccountConnectionManager stopped all connections") + + def get_connection(self, name: str = "default") -> Any | None: + return self._connections.get(name) + + async def send_to_account(self, name: str, *args, **kwargs) -> Any: + connection = self._connections.get(name) + if connection is None: + raise RuntimeError(f"Account '{name}' not connected") + if hasattr(connection, "send"): + return await connection.send(*args, **kwargs) + raise RuntimeError(f"Connection for account '{name}' does not support send()") + + async def health_check_all(self) -> dict[str, Any]: + result: dict[str, Any] = {} + for name, connection in self._connections.items(): + if hasattr(connection, "health_check"): + try: + result[name] = await connection.health_check() + except Exception as e: + result[name] = {"status": "unhealthy", "error": str(e)} + else: + result[name] = {"status": "unknown"} + return result diff --git a/backend/package/yuxi/channels/adapters/urbit/adapter.py b/backend/package/yuxi/channels/adapters/urbit/adapter.py new file mode 100644 index 00000000..0ce12d91 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/adapter.py @@ -0,0 +1,913 @@ +from __future__ import annotations + +import json +import os +from collections.abc import AsyncIterator +from typing import Any + +from yuxi.channels.base import BaseChannelAdapter +from yuxi.channels.capabilities import ChannelCapabilities +from yuxi.channels.meta import ChannelMeta +from yuxi.channels.infra.circuit_breaker import CircuitBreaker, CircuitBreakerOpenError +from yuxi.channels.exceptions import ( + ChannelAuthenticationError, + ChannelConnectionError, +) +from yuxi.channels.models import ( + ChannelMessage, + ChannelResponse, + ChannelStatus, + ChannelType, + DeliveryResult, + HealthStatus, +) +from yuxi.channels.registry import register_builtin_adapter +from yuxi.utils.datetime_utils import utc_now_naive +from yuxi.utils.logging_config import logger + +from .auth import authenticate_ship +from .client import UrbitClient +from .format import build_channel_path, build_dm_channel_id, build_poke_payload +from .monitor import SSEDeduplicator, UrbitSSEManager, _graph_update_to_channel_message +from .probe import probe_ship +from .rate_limiter import RateLimiter +from .send import ( + send_poke, + send_reaction_poke, + send_del_reaction_poke, + send_edit_poke, + send_delete_poke, + send_unblock_ship_poke, + _send_media_poke, +) +from .session import resolve_session_route, detect_unsafe_session +from .approval import ApprovalSystem +from .accounts import AccountManager +from .invites import InviteManager +from .threads import ThreadManager +from .authorization import ChannelAuthorization +from .settings import SettingsStore +from .discovery import ChannelDiscovery +from .history import MessageCache +from .summarizer import Summarizer + + +@register_builtin_adapter +class UrbitAdapter(BaseChannelAdapter): + channel_id = "urbit" + channel_type = ChannelType.URBIT + + text_chunk_limit = 10000 + supports_markdown = False + supports_streaming = True + streaming_modes = ["off", "partial"] + max_media_size_mb = 100 + + # reactions, edit, unsend 为 ForcePilot 独有增强能力（OpenClaw/Tlon 不支持） + capabilities = ChannelCapabilities( + chat_types=["direct", "group", "thread"], + media=True, + reply=True, + reactions=True, + edit=True, + unsend=True, + threads=True, + supports_markdown=False, + supports_streaming=True, + streaming_modes=["off", "partial"], + text_chunk_limit=10000, + max_media_size_mb=100, + ) + meta = ChannelMeta(id="urbit", label="Urbit") + + def __init__(self, config: dict[str, Any] | None = None): + super().__init__(config) + self._status = ChannelStatus.DISCONNECTED + self._client: UrbitClient | None = None + self._ship_code: str = "" + self._ship_url: str = "" + self._circuit_breaker = CircuitBreaker(failure_threshold=5) + self._sse_manager: UrbitSSEManager | None = None + self._active_subscriptions: set[str] = set() + self._dedup = SSEDeduplicator() + rate_limit_config = self.config.get("rate_limit", {}) + self._rate_limiter = RateLimiter( + limit=rate_limit_config.get("max_per_minute", 30), + window=rate_limit_config.get("window_s", 60.0), + ) + self._approval_system = ApprovalSystem( + owner_ship=self.config.get("owner_ship", ""), + ) + self._account_manager = AccountManager(self.config.get("accounts")) + self._invite_manager = InviteManager( + auto_accept_groups=self.config.get("autoAcceptGroupInvites", False), + group_invite_allowlist=self.config.get("groupInviteAllowlist", []), + auto_accept_dm=self.config.get("autoAcceptDmInvites", False), + dm_allowlist=self.config.get("dm_allowlist", []), + ) + self._thread_manager = ThreadManager( + max_history=self.config.get("thread", {}).get("max_history", 20), + context_lines=self.config.get("thread", {}).get("context_lines", 10), + ) + self._channel_auth = ChannelAuthorization(self.config) + self._settings_store: SettingsStore | None = None + self._session_warning_sent: set[str] = set() + self._channel_discovery: ChannelDiscovery | None = None + self._message_cache = MessageCache() + self._summarizer = Summarizer(self._message_cache) + + @property + def status(self) -> str: + return self._status.value + + def _resolve_ship_code(self) -> str: + return self.config.get("ship_code", "") or os.environ.get("URBIT_SHIP_CODE", "") + + async def _check_rate_limit(self) -> DeliveryResult | None: + if not await self._rate_limiter.wait_and_acquire(timeout=5.0): + return DeliveryResult(success=False, error="Rate limit exceeded, please retry later") + return None + + async def pre_connect(self) -> dict[str, Any]: + ship_url = self.config.get("ship_url", "") or os.environ.get("URBIT_SHIP_URL", "") + ship_name = self.config.get("ship_name", "").lstrip("~") or os.environ.get("URBIT_SHIP_NAME", "").lstrip("~") + errors: list[str] = [] + + if not ship_url: + errors.append("ship_url is required in config") + elif not ship_url.startswith(("http://", "https://")): + errors.append(f"ship_url must start with http:// or https://, got: {ship_url}") + + if not ship_name: + errors.append("ship_name is required in config") + + ship_code = self._resolve_ship_code() + if not ship_code: + errors.append("ship_code is required (set URBIT_SHIP_CODE env or config.ship_code)") + + if errors: + raise ChannelAuthenticationError("; ".join(errors)) + + return {"ship_url": ship_url, "ship_name": ship_name} + + async def _do_reauth(self) -> None: + if self._client and self._ship_code: + from .auth import authenticate_ship + + await authenticate_ship(self._client, self._ship_code) + + async def connect(self) -> None: + if self._status == ChannelStatus.CONNECTED: + return + + self._status = ChannelStatus.CONNECTING + logger.info(f"[Urbit] Starting channel '{self.channel_id}'...") + + try: + self._ship_url = self.config.get("ship_url", "http://localhost:8080") or os.environ.get( + "URBIT_SHIP_URL", "http://localhost:8080" + ) + ship_name = self.config.get("ship_name", "").lstrip("~") or os.environ.get("URBIT_SHIP_NAME", "").lstrip( + "~" + ) + if not ship_name: + raise ChannelAuthenticationError("ship_name is required in config") + self._ship_code = self._resolve_ship_code() + if not self._ship_code: + raise ChannelAuthenticationError("ship_code is required (set URBIT_SHIP_CODE env or config.ship_code)") + + from .doctor import create_legacy_private_network_doctor_contract, apply_doctor_migrations + + doctor_migrations = create_legacy_private_network_doctor_contract(self.config) + if doctor_migrations: + self.config = apply_doctor_migrations(self.config, doctor_migrations) + + self._client = UrbitClient( + ship_url=self._ship_url, + ship_name=ship_name, + timeout=self.config.get("timeout", 15.0), + ) + await self._client.start() + + await authenticate_ship(self._client, self._ship_code) + + await probe_ship(self._client) + + if self._settings_store is None and self._client: + self._settings_store = SettingsStore(self._client) + await self._settings_store.load(self.config) + + migrations = await self._settings_store.build_settings_migrations() + if migrations: + logger.info(f"[Urbit] Running {len(migrations)} config→Settings Store migrations...") + for mig in migrations: + await self._settings_store.poke_migration(mig["bucket_key"], mig["entry_key"], mig["value"]) + + self._apply_settings_to_runtime() + await self._approval_system.load_from_settings_store(self._client) + + self._sse_manager = UrbitSSEManager( + client=self._client, + ship_url=self._ship_url, + on_message=self._handle_sse_message, + status_check=lambda: self._status == ChannelStatus.CONNECTED, + reconnect_delay=self.config.get("sse", {}).get("reconnect_delay_s", 5), + dedup=self._dedup, + bot_ship_name=ship_name, + reauth_callback=self._do_reauth, + thread_manager=self._thread_manager, + message_cache=self._message_cache, + ) + await self._sse_manager.start( + { + "chat": "/~/channel/chat-store", + "channels": "/~/channel/channels-server", + "contacts": "/~/channel/contacts-store", + "foreigns": "/~/channel/groups-store", + "groups-ui": "/~/channel/groups-ui", + "settings": "/~/channel/settings-store", + } + ) + self._sse_manager.set_foreign_update_handler(self._invite_manager.handle_foreign_update) + self._sse_manager.set_settings_update_handler(self._handle_settings_update) + self._sse_manager.set_groups_ui_update_handler(self._handle_groups_ui_update) + + self._channel_discovery = ChannelDiscovery( + client=self._client, + auto_discover=self.config.get("autoDiscoverChannels", False), + group_channels=self.config.get("groupChannels"), + settings_store=self._settings_store, + ) + await self._channel_discovery.start() + + self._status = ChannelStatus.CONNECTED + logger.info( + f"[Urbit] Channel '{self.channel_id}' connected to {self._ship_url} (Ship: ~{self._client.ship_name})" + ) + + except ChannelAuthenticationError: + self._status = ChannelStatus.ERROR + raise + except ChannelConnectionError: + self._status = ChannelStatus.ERROR + raise + except Exception as e: + self._status = ChannelStatus.ERROR + logger.error(f"[Urbit] Failed to start channel '{self.channel_id}': {e}") + raise + + async def disconnect(self) -> None: + if self._status == ChannelStatus.DISCONNECTED: + return + + logger.info(f"[Urbit] Stopping channel '{self.channel_id}'...") + + try: + if self._channel_discovery: + await self._channel_discovery.stop() + self._channel_discovery = None + + if self._sse_manager: + await self._sse_manager.stop() + self._sse_manager = None + + if self._client: + await self._client.stop() + self._client = None + + self._active_subscriptions.clear() + + self._status = ChannelStatus.DISCONNECTED + logger.info(f"[Urbit] Channel '{self.channel_id}' stopped") + + except Exception as e: + logger.error(f"[Urbit] Error stopping channel '{self.channel_id}': {e}") + self._status = ChannelStatus.ERROR + + async def send(self, response: ChannelResponse) -> DeliveryResult: + if not self._client: + return DeliveryResult(success=False, error="Client not initialized") + if self._status != ChannelStatus.CONNECTED: + return DeliveryResult(success=False, error="Not connected") + + if rate_limit_result := await self._check_rate_limit(): + return rate_limit_result + + response.metadata.setdefault("host_ship", self._client.ship_name) + + retry_config = self.config.get("retry", {}) + + if len(response.content) > self.text_chunk_limit: + return await self._send_chunked(response, retry_config) + + try: + return await self._circuit_breaker.call( + lambda: send_poke(self._client, response, retry_config, ship_code=self._ship_code) + ) + except CircuitBreakerOpenError: + return DeliveryResult( + success=False, + error="Circuit breaker is open, Urbit Ship unavailable", + ) + except Exception as e: + return DeliveryResult(success=False, error=str(e)) + + async def _send_chunked(self, response: ChannelResponse, retry_config: dict[str, Any]) -> DeliveryResult: + content = response.content + chunk_size = self.text_chunk_limit + chunks: list[str] = [] + + while content: + chunks.append(content[:chunk_size]) + content = content[chunk_size:] + + last_result = DeliveryResult(success=False, error="No chunks sent") + for i, chunk in enumerate(chunks): + chunk_response = ChannelResponse( + identity=response.identity, + content=chunk, + message_type=response.message_type, + attachments=response.attachments if i == len(chunks) - 1 else [], + reply_to_message_id=response.reply_to_message_id if i == 0 else None, + metadata={**response.metadata, "chunk_index": i, "chunk_total": len(chunks)}, + ) + try: + result = await self._circuit_breaker.call( + lambda cr=chunk_response: send_poke(self._client, cr, retry_config, ship_code=self._ship_code) + ) + except CircuitBreakerOpenError: + return DeliveryResult( + success=False, + error="Circuit breaker is open during chunked send", + ) + except Exception as e: + result = DeliveryResult(success=False, error=str(e)) + + if result.success: + last_result = result + elif i == 0: + return result + elif i < len(chunks) - 1: + logger.warning(f"[Urbit] Chunk {i + 1}/{len(chunks)} failed: {result.error}") + + return last_result + + async def send_reaction(self, chat_id: str, msg_id: str, emoji: str) -> DeliveryResult: + if not self._client or self._status != ChannelStatus.CONNECTED: + return DeliveryResult(success=False, error="Not connected") + + metadata = {"chat_type": "group", "host_ship": self._client.ship_name} + retry_config = self.config.get("retry", {}) + + try: + return await self._circuit_breaker.call( + lambda: send_reaction_poke( + self._client, + chat_id, + msg_id, + emoji, + metadata, + retry_config, + ship_code=self._ship_code, + ) + ) + except CircuitBreakerOpenError: + return DeliveryResult( + success=False, + error="Circuit breaker is open, Urbit Ship unavailable", + ) + except Exception as e: + return DeliveryResult(success=False, error=str(e)) + + async def send_del_reaction(self, chat_id: str, msg_id: str, emoji: str) -> DeliveryResult: + if not self._client or self._status != ChannelStatus.CONNECTED: + return DeliveryResult(success=False, error="Not connected") + + metadata = {"chat_type": "group", "host_ship": self._client.ship_name} + retry_config = self.config.get("retry", {}) + + try: + return await self._circuit_breaker.call( + lambda: send_del_reaction_poke( + self._client, + chat_id, + msg_id, + emoji, + metadata, + retry_config, + ship_code=self._ship_code, + ) + ) + except CircuitBreakerOpenError: + return DeliveryResult( + success=False, + error="Circuit breaker is open, Urbit Ship unavailable", + ) + except Exception as e: + return DeliveryResult(success=False, error=str(e)) + + async def edit_message(self, chat_id: str, msg_id: str, content: str) -> DeliveryResult: + if not self._client or self._status != ChannelStatus.CONNECTED: + return DeliveryResult(success=False, error="Not connected") + + metadata = {"chat_type": "group", "host_ship": self._client.ship_name} + retry_config = self.config.get("retry", {}) + + try: + return await self._circuit_breaker.call( + lambda: send_edit_poke( + self._client, + chat_id, + msg_id, + content, + metadata, + retry_config, + ship_code=self._ship_code, + ) + ) + except CircuitBreakerOpenError: + return DeliveryResult( + success=False, + error="Circuit breaker is open, Urbit Ship unavailable", + ) + except Exception as e: + return DeliveryResult(success=False, error=str(e)) + + async def delete_message(self, chat_id: str, msg_id: str) -> DeliveryResult: + if not self._client or self._status != ChannelStatus.CONNECTED: + return DeliveryResult(success=False, error="Not connected") + + metadata = {"chat_type": "group", "host_ship": self._client.ship_name} + retry_config = self.config.get("retry", {}) + + try: + return await self._circuit_breaker.call( + lambda: send_delete_poke( + self._client, + chat_id, + msg_id, + metadata, + retry_config, + ship_code=self._ship_code, + ) + ) + except CircuitBreakerOpenError: + return DeliveryResult( + success=False, + error="Circuit breaker is open, Urbit Ship unavailable", + ) + except Exception as e: + return DeliveryResult(success=False, error=str(e)) + + async def send_media(self, chat_id: str, media_type: str, data: Any) -> DeliveryResult: + if not self._client or self._status != ChannelStatus.CONNECTED: + return DeliveryResult(success=False, error="Not connected") + + if rate_limit_result := await self._check_rate_limit(): + return rate_limit_result + + from .format import build_channel_path, build_poke_payload, build_media_content + + metadata: dict[str, str] = { + "chat_type": "group", + "group_name": chat_id, + "host_ship": self._client.ship_name, + } + channel_path = build_channel_path(**{**metadata, "chat_type": "group"}) + host_ship = self._client.ship_name + media = build_media_content(media_type, str(data)) + payload = build_poke_payload( + host_ship=host_ship, + content="", + channel_path=channel_path, + media_content=media, + ) + + response = ChannelResponse( + identity=self._build_stream_identity(chat_id, ""), + content="", + metadata=metadata, + ) + retry_config = self.config.get("retry", {}) + + try: + return await self._circuit_breaker.call( + lambda: _send_media_poke(self._client, response, payload, retry_config) + ) + except CircuitBreakerOpenError: + return DeliveryResult( + success=False, + error="Circuit breaker is open, Urbit Ship unavailable", + ) + except Exception as e: + return DeliveryResult(success=False, error=str(e)) + + async def receive(self) -> AsyncIterator[ChannelMessage]: + return + yield # type: ignore[misc] + + def normalize_inbound(self, raw: dict[str, Any]) -> ChannelMessage: + return _graph_update_to_channel_message(raw, self._client.ship_name if self._client else "unknown") + + def format_outbound(self, response: ChannelResponse) -> dict[str, Any]: + chat_type = response.metadata.get("chat_type", "group") + host_ship = response.metadata.get("host_ship", "") + + channel_path = build_channel_path(**{**response.metadata, "chat_type": chat_type}) + + return build_poke_payload( + host_ship=host_ship, + content=response.content, + channel_path=channel_path, + continuation=response.metadata.get("continuation", False), + reply_to=response.reply_to_message_id, + chat_type=chat_type, + ) + + async def health_check(self) -> HealthStatus: + if not self._client: + return HealthStatus( + status="unhealthy", + last_error="Client not initialized", + metadata={"adapter_status": self._status.value}, + ) + + try: + await probe_ship(self._client) + return HealthStatus( + status="healthy", + metadata={ + "ship_url": self._ship_url, + "ship_name": f"~{self._client.ship_name}", + "adapter_status": self._status.value, + }, + last_connected_at=utc_now_naive(), + ) + except ChannelConnectionError as e: + return HealthStatus( + status="degraded", + last_error=str(e), + metadata={"adapter_status": self._status.value}, + ) + except Exception as e: + return HealthStatus( + status="unhealthy", + last_error=str(e), + metadata={"adapter_status": self._status.value}, + ) + + async def send_stream_chunk(self, chat_id: str, msg_id: str, chunk: str, finished: bool) -> DeliveryResult: + streaming_mode = self.config.get("streaming", "off") + if streaming_mode == "off": + if finished: + identity = self._build_stream_identity(chat_id, msg_id) + response = ChannelResponse( + identity=identity, + content=chunk, + metadata={"chat_type": "group", "host_ship": self._client.ship_name if self._client else ""}, + ) + return await self.send(response) + return DeliveryResult(success=True) + + identity = self._build_stream_identity(chat_id, msg_id) + response = ChannelResponse( + identity=identity, + content=chunk, + metadata={ + "chat_type": "group", + "host_ship": self._client.ship_name if self._client else "", + "continuation": not finished, + }, + ) + return await self.send(response) + + async def _handle_sse_message(self, message: ChannelMessage) -> None: + dm_policy = self.config.get("dm_policy", "pairing") + group_policy = self.config.get("group_policy", "open") + + chat_type = message.metadata.get("chat_type", "group") + sender_ship = message.metadata.get("urbit_ship", "") + channel_chat_id = message.identity.channel_chat_id + + if self._approval_system.is_ship_blocked(sender_ship): + logger.debug(f"[Urbit] Ignoring message from blocked ship ~{sender_ship}") + return + + owner_ship = self.config.get("owner_ship", "").lstrip("~") + if message.content and message.content.strip().startswith("/"): + is_owner = owner_ship and sender_ship.lower() == owner_ship.lower() + if not is_owner: + logger.info( + f"[Urbit] Slash command rejected: " + f"~{sender_ship} [user] in {channel_chat_id}: {message.content[:80]}" + ) + return + role_tag = "owner" if is_owner else "user" + message.content = f"~{sender_ship} [{role_tag}] in {channel_chat_id}:\n{message.content}" + logger.debug(f"[Urbit] Slash command authorized: ~{sender_ship} [{role_tag}]") + + if chat_type == "direct": + unsafe_participants = detect_unsafe_session(message) + if unsafe_participants: + owner_ship = self.config.get("owner_ship", "").lstrip("~") + session_id = f"{sender_ship}:{channel_chat_id}" + if owner_ship and session_id not in self._session_warning_sent: + self._session_warning_sent.add(session_id) + await self._send_dm_to_ship( + owner_ship, + f"[Unsafe Session Alert] DM session has " + f"{len(unsafe_participants)} participants: " + f"{', '.join('~' + p for p in unsafe_participants)}", + ) + + if self._approval_system.is_approval_command(message.content): + owner_ship = self.config.get("owner_ship", "").lstrip("~") + if owner_ship and sender_ship.lower() == owner_ship.lower(): + result = self._approval_system.handle_admin_command(message.content, sender_ship) + if result: + await self._send_dm_to_ship(owner_ship, result) + + content_lower = message.content.strip().lower() + if content_lower.startswith("unblock "): + ship = message.content.strip().split(" ", 1)[1].strip() + await send_unblock_ship_poke(self._client, ship, ship_code=self._ship_code) + return + + if not self._check_dm_policy(sender_ship, dm_policy): + logger.warning(f"[Urbit] DM from ~{sender_ship} blocked by dm_policy='{dm_policy}'") + await self._maybe_create_approval(sender_ship, message) + return + + if chat_type == "group": + if not self._channel_auth.is_channel_allowed(channel_chat_id, sender_ship): + logger.warning(f"[Urbit] Channel message from ~{sender_ship} blocked in {channel_chat_id}") + await self._maybe_create_channel_approval(sender_ship, message) + return + + if not self._check_group_policy(sender_ship, group_policy): + logger.warning(f"[Urbit] Group message from ~{sender_ship} blocked by group_policy='{group_policy}'") + return + + message.metadata["session_route"] = resolve_session_route(message) + + summary_info = self._summarizer.auto_detect_and_respond(channel_chat_id, message.content) + if summary_info: + message.metadata["summarization"] = summary_info + + await self._handle_message(message) + + async def _maybe_create_approval(self, sender_ship: str, message: ChannelMessage) -> None: + owner_ship = self.config.get("owner_ship", "").lstrip("~") + if not owner_ship: + return + + channel_id = message.identity.channel_chat_id + if self._approval_system.has_duplicate_pending("dm", sender_ship, channel_id): + logger.debug(f"[Urbit] Duplicate DM approval request from ~{sender_ship}, skipping") + return + + approval = self._approval_system.create_pending( + approval_type="dm", + requester_ship=sender_ship, + channel_id=channel_id, + message_content=message.content, + ) + + notification = self._approval_system.build_owner_notification(approval) + instructions = self._approval_system.build_approval_instructions(approval.id) + combined = f"{notification}\n\n{instructions}" + + await self._send_dm_to_ship(owner_ship, combined) + + async def _maybe_create_channel_approval(self, sender_ship: str, message: ChannelMessage) -> None: + owner_ship = self.config.get("owner_ship", "").lstrip("~") + if not owner_ship: + return + + channel_id = message.identity.channel_chat_id + if self._approval_system.has_duplicate_pending("channel", sender_ship, channel_id): + logger.debug(f"[Urbit] Duplicate channel approval request from ~{sender_ship}, skipping") + return + + approval = self._approval_system.create_pending( + approval_type="channel", + requester_ship=sender_ship, + channel_id=channel_id, + message_content=message.content, + ) + + notification = self._approval_system.build_owner_notification(approval) + instructions = self._approval_system.build_approval_instructions(approval.id) + combined = f"[Channel Message Approval]\n{notification}\n\n{instructions}" + + await self._send_dm_to_ship(owner_ship, combined) + + async def _handle_settings_update(self, settings_update: dict[str, Any]) -> None: + if not self._settings_store: + return + + action = settings_update.get("action", "") + bucket_key = settings_update.get("bucket_key", "") + entry_key = settings_update.get("entry_key", "") + value = settings_update.get("value") + + try: + if action == "put" and value is not None: + self._settings_store.merge_key(bucket_key, {entry_key: value}) + prev = self._settings_store.get(bucket_key, {}) + if isinstance(prev, dict): + merged = {**prev, entry_key: value} + else: + merged = {entry_key: value} + store = self._settings_store.get_all() + store[bucket_key] = merged + self._settings_store._store = store + logger.debug(f"[Urbit] Settings merged: {bucket_key}/{entry_key}") + elif action == "del": + store_data = self._settings_store.get_all() + bucket = store_data.get(bucket_key) + if isinstance(bucket, dict): + bucket.pop(entry_key, None) + self._settings_store._store = store_data + logger.debug(f"[Urbit] Settings removed: {bucket_key}/{entry_key}") + + self._apply_settings_to_runtime() + except Exception as e: + logger.warning(f"[Urbit] Settings merge error: {e}") + + def _apply_settings_to_runtime(self) -> None: + if not self._settings_store: + return + store_data = self._settings_store.get_all() + + dm_allowlist = store_data.get("dmAllowlist") + if dm_allowlist is not None and isinstance(dm_allowlist, list): + self.config["dm_allowlist"] = dm_allowlist + + group_channels = store_data.get("groupChannels") + if group_channels is not None and isinstance(group_channels, list): + self.config["groupChannels"] = group_channels + if self._channel_discovery: + for ch in group_channels: + self._channel_discovery.add_channel(ch) + + channel_rules_raw = store_data.get("channelRules") + if channel_rules_raw is not None: + if isinstance(channel_rules_raw, str): + try: + channel_rules = json.loads(channel_rules_raw) + except json.JSONDecodeError: + logger.warning("[Urbit] Failed to parse channelRules JSON") + channel_rules = None + else: + channel_rules = channel_rules_raw + if isinstance(channel_rules, dict): + self.config.setdefault("authorization", {}) + self.config["authorization"]["channel_rules"] = channel_rules + self._channel_auth = ChannelAuthorization(self.config) + + pending_approvals_raw = store_data.get("pendingApprovals") + if pending_approvals_raw is not None: + if isinstance(pending_approvals_raw, str): + try: + pending_approvals = json.loads(pending_approvals_raw) + except json.JSONDecodeError: + pending_approvals = None + else: + pending_approvals = pending_approvals_raw + if isinstance(pending_approvals, list): + for item in pending_approvals: + if isinstance(item, dict) and item.get("status") == "pending": + sid = item.get("id", "") + if sid and not self._approval_system._pending.get(sid): + from .approval import PendingApproval + + approval = PendingApproval( + id=sid, + approval_type=item.get("type", "dm"), + requester_ship=item.get("requester", ""), + channel_id=item.get("channel", ""), + message_content=item.get("content", ""), + created_at=item.get("created_at", 0), + status="pending", + ) + self._approval_system._pending[sid] = approval + + logger.debug("[Urbit] Runtime config updated from Settings Store hot-reload") + + async def _handle_groups_ui_update(self, update: dict[str, Any]) -> None: + action = update.get("action", "") + group_path = update.get("group_path", "") or update.get("group_id", "") + + if action in ("add", "join") and group_path: + logger.info(f"[Urbit] Groups-UI {action}: {group_path}") + channels = update.get("channels", []) + if isinstance(channels, list): + for ch in channels: + if isinstance(ch, str) and ch.startswith("chat/") and self._channel_discovery: + self._channel_discovery.add_channel(ch) + if self._channel_discovery: + try: + await self._channel_discovery.persist_group_channels() + except Exception as e: + logger.warning(f"[Urbit] Failed to persist group channels after groups-ui update: {e}") + + if action == "kick" and group_path and self._channel_discovery: + logger.info(f"[Urbit] Groups-UI kick: {group_path}") + + async def _send_dm_to_ship(self, ship: str, content: str) -> None: + if not self._client: + return + from yuxi.channels.models import ChannelIdentity + + identity = ChannelIdentity( + channel_id="urbit", + channel_type=ChannelType.URBIT, + channel_user_id=f"~{ship}", + channel_chat_id=build_dm_channel_id(ship, self._client.ship_name), + ) + fake_response = ChannelResponse( + identity=identity, + content=content, + metadata={"chat_type": "direct", "host_ship": self._client.ship_name, "target_ship": ship}, + ) + try: + await send_poke(self._client, fake_response, {}, ship_code=self._ship_code) + except Exception as e: + logger.error(f"[Urbit] Failed to send DM to ~{ship}: {e}") + + def _check_dm_policy(self, sender_ship: str, policy: str) -> bool: + owner_ship = self.config.get("owner_ship", "").lstrip("~") + if owner_ship and sender_ship.lower() == owner_ship.lower(): + return True + + match policy: + case "open": + return True + case "disabled": + return False + case "pairing": + return True + case "allowlist": + allow_from = self.config.get("dm_allowlist", []) or self.config.get("allow_from", []) + return f"urbit:{sender_ship}" in allow_from + case _: + return False + + async def fetch_thread_history(self, resource_path: str, parent_id: str) -> list[dict[str, Any]]: + if not self._client or not self._thread_manager: + return [] + return await self._thread_manager.fetch_thread_history( + self._client, + self._client.ship_name, + resource_path, + parent_id, + ) + + def build_thread_context(self, history: list[dict[str, Any]]) -> str: + if not self._thread_manager: + return "" + return self._thread_manager.build_thread_context(history) + + async def get_user_info(self, channel_user_id: str) -> dict[str, Any]: + ship = channel_user_id.lstrip("~") + return { + "id": f"urbit:{ship}", + "username": ship, + "display_name": f"~{ship}", + "channel_type": self.channel_type.value, + } + + async def download_media(self, file_id: str) -> bytes: + from .probe import validate_urbit_url + + is_valid, warnings = validate_urbit_url(file_id) + if not is_valid: + raise ValueError(f"SSRF blocked: invalid URL - {'; '.join(warnings)}") + + import httpx + + url = file_id + async with httpx.AsyncClient(timeout=httpx.Timeout(30.0)) as client: + r = await client.get(url) + r.raise_for_status() + return r.content + + def _check_group_policy(self, sender_ship: str, policy: str) -> bool: + match policy: + case "open": + return True + case "disabled": + return False + case "allowlist": + group_allow = self.config.get("group_allow_from", []) + if f"urbit:{sender_ship}" in group_allow: + return True + groups_config = self.config.get("groups", {}) + for group_config in groups_config.values(): + per_allow = group_config.get("allow_from", []) + if f"urbit:{sender_ship}" in per_allow: + return True + return False + case _: + return False diff --git a/backend/package/yuxi/channels/adapters/urbit/approval.py b/backend/package/yuxi/channels/adapters/urbit/approval.py new file mode 100644 index 00000000..fc4a8a86 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/approval.py @@ -0,0 +1,259 @@ +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from typing import Any + +from yuxi.utils.logging_config import logger + + +@dataclass +class PendingApproval: + id: str + approval_type: str + requester_ship: str + channel_id: str + message_content: str + created_at: float + status: str = "pending" + + def approve(self) -> None: + self.status = "approved" + + def deny(self) -> None: + self.status = "denied" + + +class ApprovalSystem: + def __init__(self, owner_ship: str = ""): + self._owner_ship = owner_ship.lstrip("~") + self._pending: dict[str, PendingApproval] = {} + self._resolved: dict[str, PendingApproval] = {} + self._blocked_ships: set[str] = set() + + @property + def owner_ship(self) -> str: + return self._owner_ship + + def create_pending( + self, + approval_type: str, + requester_ship: str, + channel_id: str, + message_content: str, + ) -> PendingApproval: + import time + + approval = PendingApproval( + id=f"{approval_type}-{int(time.time())}-{str(uuid.uuid4())[:8]}", + approval_type=approval_type, + requester_ship=requester_ship, + channel_id=channel_id, + message_content=message_content, + created_at=time.time(), + ) + self._pending[approval.id] = approval + logger.info(f"[Urbit] Approval {approval.id} created: {approval_type} from ~{requester_ship}") + return approval + + def approve(self, approval_id: str) -> PendingApproval | None: + approval = self._pending.pop(approval_id, None) + if approval: + approval.approve() + self._resolved[approval_id] = approval + logger.info(f"[Urbit] Approval {approval_id} approved") + return approval + + def deny(self, approval_id: str) -> PendingApproval | None: + approval = self._pending.pop(approval_id, None) + if approval: + approval.deny() + self._resolved[approval_id] = approval + logger.info(f"[Urbit] Approval {approval_id} denied") + return approval + + def get_pending(self) -> list[PendingApproval]: + return list(self._pending.values()) + + def get_pending_count(self) -> int: + return len(self._pending) + + def block_ship(self, ship: str) -> bool: + ship = ship.lstrip("~").lower() + if ship in self._blocked_ships: + return False + self._blocked_ships.add(ship) + logger.info(f"[Urbit] Ship blocked: ~{ship}") + return True + + def unblock_ship(self, ship: str) -> bool: + ship = ship.lstrip("~").lower() + if ship not in self._blocked_ships: + return False + self._blocked_ships.discard(ship) + logger.info(f"[Urbit] Ship unblocked: ~{ship}") + return True + + def get_blocked_ships(self) -> list[str]: + return sorted(self._blocked_ships) + + def is_ship_blocked(self, ship: str) -> bool: + return ship.lstrip("~").lower() in self._blocked_ships + + def has_duplicate_pending(self, approval_type: str, requester_ship: str, channel_id: str) -> bool: + req_ship = requester_ship.lstrip("~").lower() + for p in self._pending.values(): + if ( + p.approval_type == approval_type + and p.requester_ship.lstrip("~").lower() == req_ship + and p.channel_id == channel_id + and p.status == "pending" + ): + return True + return False + + def build_owner_notification(self, approval: PendingApproval) -> str: + return ( + f"New {approval.approval_type} request from ~{approval.requester_ship}" + f" (ID: {approval.id}): {approval.message_content[:200]}" + ) + + def build_approval_instructions(self, approval_id: str) -> str: + return ( + f"To approve: send `approve {approval_id}`\n" + f"To deny: send `deny {approval_id}`\n" + f"To block: send `block {approval_id}`\n" + f"To list pending: send `pending`\n" + f"To list blocked: send `blocked`" + ) + + def is_approval_command(self, content: str) -> bool: + return any( + content.strip().startswith(cmd) + for cmd in ("approve ", "deny ", "block ", "unblock ", "approvals", "pending", "blocked") + ) + + def handle_admin_command(self, content: str, sender_ship: str) -> str | None: + if not self._owner_ship or sender_ship.lower() != self._owner_ship.lower(): + return None + + content = content.strip() + + if content.startswith("approve "): + aid = content.split(" ", 1)[1].strip() + approval = self.approve(aid) + if approval: + return f"Approved {aid} (~{approval.requester_ship})" + return f"Approval {aid} not found" + + if content.startswith("deny "): + aid = content.split(" ", 1)[1].strip() + approval = self.deny(aid) + if approval: + return f"Denied {aid} (~{approval.requester_ship})" + return f"Approval {aid} not found" + + if content.startswith("block "): + aid = content.split(" ", 1)[1].strip() + approval = self._pending.get(aid) + if approval: + self.block_ship(approval.requester_ship) + self.deny(aid) + return f"Blocked and denied {aid} (~{approval.requester_ship})" + return f"Approval {aid} not found" + + if content.startswith("unblock "): + ship = content.split(" ", 1)[1].strip() + if self.unblock_ship(ship): + return f"Unblocked ~{ship}" + return f"Ship ~{ship} is not blocked" + + if content in ("approvals", "pending"): + pending = self.get_pending() + if not pending: + return "No pending approvals" + lines = ["Pending approvals:"] + for p in pending: + lines.append(f" {p.id}: {p.approval_type} from ~{p.requester_ship}") + return "\n".join(lines) + + if content == "blocked": + blocked = self.get_blocked_ships() + if not blocked: + return "No blocked ships" + lines = ["Blocked ships:"] + for ship in blocked: + lines.append(f" ~{ship}") + return "\n".join(lines) + + return None + + async def persist_to_settings_store(self, client: Any) -> None: + from .settings import SettingsStore + + store_data: dict[str, Any] = { + "pending_approvals": [], + "blocked_ships": sorted(self._blocked_ships), + } + for p in self._pending.values(): + store_data["pending_approvals"].append( + { + "id": p.id, + "type": p.approval_type, + "requester": p.requester_ship, + "channel": p.channel_id, + "content": p.message_content[:500], + "created_at": p.created_at, + "status": p.status, + } + ) + try: + store = SettingsStore(client) + await store.load() + merged = store.get_all() + merged["pending_approvals"] = store_data["pending_approvals"] + merged["blocked_ships"] = store_data["blocked_ships"] + store._store = merged + logger.debug( + f"[Urbit] Persisted {len(self._pending)} pending approvals " + f"and {len(self._blocked_ships)} blocked ships to Settings Store" + ) + except Exception as e: + logger.warning(f"[Urbit] Failed to persist approvals: {e}") + + async def load_from_settings_store(self, client: Any) -> list[PendingApproval]: + from .settings import SettingsStore + + try: + store = SettingsStore(client) + data = await store.load() + stored = data.get("pending_approvals", []) + stored_blocked = data.get("blocked_ships", []) + if isinstance(stored_blocked, list): + for ship in stored_blocked: + self._blocked_ships.add(ship.lstrip("~").lower()) + loaded: list[PendingApproval] = [] + for item in stored: + approval = PendingApproval( + id=item.get("id", str(uuid.uuid4())[:8]), + approval_type=item.get("type", "dm"), + requester_ship=item.get("requester", ""), + channel_id=item.get("channel", ""), + message_content=item.get("content", ""), + created_at=item.get("created_at", 0), + status=item.get("status", "pending"), + ) + if approval.status == "pending": + self._pending[approval.id] = approval + else: + self._resolved[approval.id] = approval + loaded.append(approval) + if loaded or stored_blocked: + logger.info( + f"[Urbit] Loaded {len(loaded)} approvals " + f"and {len(self._blocked_ships)} blocked ships from Settings Store" + ) + return loaded + except Exception as e: + logger.warning(f"[Urbit] Failed to load approvals: {e}") + return [] diff --git a/backend/package/yuxi/channels/adapters/urbit/auth.py b/backend/package/yuxi/channels/adapters/urbit/auth.py new file mode 100644 index 00000000..4d80d1c6 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/auth.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import httpx + +from yuxi.channels.exceptions import ChannelAuthenticationError, ChannelConnectionError +from yuxi.utils.logging_config import logger + +from .client import UrbitClient + + +async def authenticate_ship( + client: UrbitClient, + ship_code: str, +) -> str: + try: + r = await client.post("/~/login", json={"password": ship_code}) + except httpx.RequestError as e: + raise ChannelConnectionError("Cannot reach Urbit Ship") from e + + if r.status_code == 403: + raise ChannelAuthenticationError() + if r.status_code != 200: + raise ChannelAuthenticationError() + + cookie = r.cookies.get(client.auth_cookie_name) + if not cookie: + raise ChannelAuthenticationError() + + client.set_session_cookie(cookie) + logger.info(f"[Urbit] Authenticated to Ship ~{client.ship_name}") + return cookie + + +async def refresh_session_if_needed( + client: UrbitClient, + ship_code: str, +) -> str: + logger.info("[Urbit] Session expired, re-authenticating...") + return await authenticate_ship(client, ship_code) diff --git a/backend/package/yuxi/channels/adapters/urbit/authorization.py b/backend/package/yuxi/channels/adapters/urbit/authorization.py new file mode 100644 index 00000000..72d80ca5 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/authorization.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from typing import Any + +from yuxi.utils.logging_config import logger + + +class ChannelAuthorization: + def __init__(self, config: dict[str, Any] | None = None): + config = config or {} + auth_config = config.get("authorization", {}) + self._channel_rules: dict[str, dict[str, Any]] = auth_config.get("channel_rules", {}) + self._default_authorized_ships: list[str] = auth_config.get("default_authorized_ships", []) + self._group_allow_from: list[str] = config.get("group_allow_from", []) + + def is_channel_allowed(self, channel_id: str, sender_ship: str) -> bool: + sender_id = f"urbit:{sender_ship}" + + if sender_id in self._default_authorized_ships: + return True + if sender_id in self._group_allow_from: + return True + + rule = self._channel_rules.get(channel_id) + if rule is None: + return True + + policy = rule.get("policy", "open") + match policy: + case "open": + return True + case "disabled": + return False + case "allowlist": + allow_from = rule.get("allow_from", []) + return sender_id in allow_from + case _: + return False + + def get_channel_policy(self, channel_id: str) -> str: + rule = self._channel_rules.get(channel_id) + if rule: + return rule.get("policy", "open") + return "open" + + def add_channel_rule( + self, + channel_id: str, + policy: str = "allowlist", + allow_from: list[str] | None = None, + ) -> None: + self._channel_rules[channel_id] = { + "policy": policy, + "allow_from": allow_from or [], + } + logger.info(f"[Urbit] Channel rule added: {channel_id} → {policy}") + + def remove_channel_rule(self, channel_id: str) -> None: + self._channel_rules.pop(channel_id, None) diff --git a/backend/package/yuxi/channels/adapters/urbit/cites.py b/backend/package/yuxi/channels/adapters/urbit/cites.py new file mode 100644 index 00000000..d28416cf --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/cites.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from typing import Any + + +def parse_cite(reference: dict[str, Any]) -> dict[str, str] | None: + cite_type = reference.get("cite") + if not cite_type: + return None + + match cite_type: + case "chan": + return { + "type": "chan", + "group": reference.get("group", ""), + "chan": reference.get("chan", ""), + } + case "group": + return { + "type": "group", + "group": reference.get("group", ""), + } + case "desk": + return { + "type": "desk", + "desk": reference.get("desk", ""), + } + case "bait": + return { + "type": "bait", + "bait": reference.get("bait", ""), + } + case _: + return None + + +def cite_to_text(cite: dict[str, str]) -> str: + match cite.get("type"): + case "chan": + return f"[channel: {cite.get('group')}/{cite.get('chan')}]" + case "group": + return f"[group: {cite.get('group')}]" + case "desk": + return f"[desk: {cite.get('desk')}]" + case "bait": + return f"[bait: {cite.get('bait')}]" + case _: + return "[unknown reference]" diff --git a/backend/package/yuxi/channels/adapters/urbit/client.py b/backend/package/yuxi/channels/adapters/urbit/client.py new file mode 100644 index 00000000..cbf9ead4 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/client.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import ipaddress +from typing import Any +from urllib.parse import urlparse + +import httpx + +from yuxi.utils.logging_config import logger + + +def _normalize_ship_url(url: str) -> str: + url = url.rstrip("/") + if not url.startswith(("http://", "https://")): + url = f"https://{url}" + + try: + parsed = urlparse(url) + hostname = parsed.hostname + if hostname: + addr = ipaddress.ip_address(hostname.strip("[]")) + if isinstance(addr, ipaddress.IPv6Address): + host_part = f"[{addr}]" if "[" not in parsed.netloc else parsed.hostname + port_part = f":{parsed.port}" if parsed.port else "" + url = f"{parsed.scheme}://{host_part}{port_part}{parsed.path}" + except (ValueError, ipaddress.AddressValueError): + pass + + return url + + +class UrbitClient: + def __init__( + self, + ship_url: str, + ship_name: str, + timeout: float = 15.0, + ): + self.ship_url = _normalize_ship_url(ship_url) + self.ship_name = ship_name.lstrip("~") + self._http: httpx.AsyncClient | None = None + self._timeout = timeout + self._session_cookie: str | None = None + + @property + def http(self) -> httpx.AsyncClient: + if self._http is None: + raise RuntimeError("UrbitClient not started. Call start() first.") + return self._http + + @property + def session_cookie(self) -> str | None: + return self._session_cookie + + @property + def auth_cookie_name(self) -> str: + return f"urbauth-~{self.ship_name}" + + async def start(self) -> None: + self._http = httpx.AsyncClient( + base_url=self.ship_url, + timeout=httpx.Timeout(self._timeout), + follow_redirects=True, + ) + logger.debug(f"[Urbit] HTTP client started for {self.ship_url}") + + async def stop(self) -> None: + if self._http: + await self._http.aclose() + self._http = None + self._session_cookie = None + logger.debug("[Urbit] HTTP client stopped") + + def set_session_cookie(self, cookie: str) -> None: + self._session_cookie = cookie + + def _cookies(self) -> dict[str, str] | None: + if self._session_cookie: + return {self.auth_cookie_name: self._session_cookie} + return None + + async def post(self, path: str, json: dict[str, Any] | None = None, **kwargs) -> httpx.Response: + return await self.http.post(path, json=json, cookies=self._cookies(), **kwargs) + + async def get(self, path: str, **kwargs) -> httpx.Response: + return await self.http.get(path, cookies=self._cookies(), **kwargs) + + async def put(self, path: str, json: dict[str, Any] | None = None, **kwargs) -> httpx.Response: + return await self.http.put(path, json=json, cookies=self._cookies(), **kwargs) + + async def delete(self, path: str, **kwargs) -> httpx.Response: + return await self.http.delete(path, cookies=self._cookies(), **kwargs) + + async def stream_get(self, path: str, timeout: float = 300.0, **kwargs) -> httpx.Response: + req = self.http.build_request("GET", self.ship_url + path, cookies=self._cookies(), **kwargs) + return await self.http.send(req, stream=True, timeout=httpx.Timeout(timeout)) diff --git a/backend/package/yuxi/channels/adapters/urbit/config_ui_hints.py b/backend/package/yuxi/channels/adapters/urbit/config_ui_hints.py new file mode 100644 index 00000000..808f79b6 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/config_ui_hints.py @@ -0,0 +1,398 @@ +from __future__ import annotations + +from typing import Any + + +def get_config_ui_hints() -> list[dict[str, Any]]: + return [ + { + "key": "ship_url", + "label": "Ship URL", + "type": "url", + "placeholder": "http://localhost:8080", + "help": "Your Urbit ship HTTP endpoint (e.g. http://localhost:8080 or https://ship.arvo.network)", + "required": True, + "category": "connection", + "order": 1, + }, + { + "key": "ship_name", + "label": "Ship Name", + "type": "text", + "placeholder": "zod", + "help": "Your Urbit ship name without the leading tilde (e.g. zod, ~zod)", # noqa: S104 + "required": True, + "category": "connection", + "order": 2, + }, + { + "key": "ship_code", + "label": "Ship +code", + "type": "password", + "placeholder": "XXXX-XXXX-XXXX-XXXX", + "help": "Your ship login code (+code) for authentication", + "required": True, + "category": "connection", + "order": 3, + }, + { + "key": "owner_ship", + "label": "Owner Ship", + "type": "text", + "placeholder": "zod", + "help": "Ship that receives admin notifications and can execute slash commands", + "required": False, + "category": "authorization", + "order": 4, + }, + { + "key": "dm_policy", + "label": "DM Policy", + "type": "select", + "options": [ + {"value": "open", "label": "Open — accept all DMs"}, + {"value": "pairing", "label": "Pairing — require pairing/approval"}, + {"value": "allowlist", "label": "Allowlist — only listed ships"}, + {"value": "disabled", "label": "Disabled — reject all DMs"}, + ], + "default": "pairing", + "help": "Controls how the bot handles incoming direct messages", + "required": False, + "category": "authorization", + "order": 5, + }, + { + "key": "group_policy", + "label": "Group Policy", + "type": "select", + "options": [ + {"value": "open", "label": "Open — accept all group messages"}, + {"value": "allowlist", "label": "Allowlist — only listed ships"}, + {"value": "disabled", "label": "Disabled — reject all group messages"}, + ], + "default": "open", + "help": "Controls how the bot handles incoming group messages", + "required": False, + "category": "authorization", + "order": 6, + }, + { + "key": "dm_allowlist", + "label": "DM Allowlist", + "type": "list", + "help": "List of ships allowed to send DMs (e.g. urbit:zod, urbit:bus)", + "required": False, + "category": "authorization", + "order": 7, + }, + { + "key": "group_channels", + "label": "Group Channels", + "type": "list", + "help": "Channels to monitor (e.g. chat/some-group, diary/some-diary)", + "required": False, + "category": "channels", + "order": 8, + }, + { + "key": "autoDiscoverChannels", + "label": "Auto-discover Channels", + "type": "boolean", + "default": True, + "help": "Automatically discover and subscribe to channels from Groups UI events", + "required": False, + "category": "channels", + "order": 9, + }, + { + "key": "autoAcceptDmInvites", + "label": "Auto Accept DM Invites", + "type": "boolean", + "default": False, + "help": "Automatically accept incoming DM invitations", + "required": False, + "category": "invites", + "order": 10, + }, + { + "key": "autoAcceptGroupInvites", + "label": "Auto Accept Group Invites", + "type": "boolean", + "default": False, + "help": "Automatically accept incoming group invitations", + "required": False, + "category": "invites", + "order": 11, + }, + { + "key": "groupInviteAllowlist", + "label": "Group Invite Allowlist", + "type": "list", + "help": "List of group paths allowed for auto-accept invites", + "required": False, + "category": "invites", + "order": 12, + }, + { + "key": "rate_limit.max_per_minute", + "label": "Rate Limit (per minute)", + "type": "number", + "default": 30, + "min": 1, + "max": 300, + "help": "Maximum number of outgoing messages per minute", + "required": False, + "category": "performance", + "order": 20, + }, + { + "key": "rate_limit.window_s", + "label": "Rate Limit Window (seconds)", + "type": "number", + "default": 60, + "min": 1, + "max": 3600, + "help": "Time window for rate limiting in seconds", + "required": False, + "category": "performance", + "order": 21, + }, + { + "key": "retry.attempts", + "label": "Retry Attempts", + "type": "number", + "default": 3, + "min": 0, + "max": 10, + "help": "Maximum number of retry attempts for failed sends", + "required": False, + "category": "performance", + "order": 22, + }, + { + "key": "retry.min_delay_ms", + "label": "Retry Min Delay (ms)", + "type": "number", + "default": 500, + "min": 100, + "max": 60000, + "help": "Minimum delay between retries in milliseconds", + "required": False, + "category": "performance", + "order": 23, + }, + { + "key": "retry.max_delay_ms", + "label": "Retry Max Delay (ms)", + "type": "number", + "default": 30000, + "min": 1000, + "max": 300000, + "help": "Maximum delay between retries in milliseconds", + "required": False, + "category": "performance", + "order": 24, + }, + { + "key": "sse.reconnect_delay_s", + "label": "SSE Reconnect Delay (s)", + "type": "number", + "default": 5, + "min": 1, + "max": 300, + "help": "Initial delay before SSE reconnection in seconds", + "required": False, + "category": "connection", + "order": 25, + }, + { + "key": "thread.max_history", + "label": "Thread Max History", + "type": "number", + "default": 20, + "min": 1, + "max": 100, + "help": "Maximum number of messages to fetch for thread history", + "required": False, + "category": "threads", + "order": 30, + }, + { + "key": "thread.context_lines", + "label": "Thread Context Lines", + "type": "number", + "default": 10, + "min": 1, + "max": 50, + "help": "Number of recent thread messages to include as context", + "required": False, + "category": "threads", + "order": 31, + }, + { + "key": "streaming", + "label": "Streaming Mode", + "type": "select", + "options": [ + {"value": "off", "label": "Off — send complete messages"}, + {"value": "partial", "label": "Partial — stream via edits"}, + ], + "default": "off", + "help": "Controls how streaming responses are delivered", + "required": False, + "category": "output", + "order": 40, + }, + { + "key": "streaming.chunkMode", + "label": "Streaming Chunk Mode", + "type": "select", + "options": [ + {"value": "auto", "label": "Auto — adaptive chunking"}, + {"value": "fixed", "label": "Fixed — constant chunk size"}, + {"value": "sentence", "label": "Sentence — split on sentence boundaries"}, + ], + "default": "auto", + "help": "How to split streaming output into chunks", + "required": False, + "category": "output", + "order": 41, + }, + { + "key": "network.dangerously_allow_private_network", + "label": "Allow Private Network", + "type": "boolean", + "default": False, + "help": "Allow connections to private/local network addresses (enables SSRF risk)", + "required": False, + "category": "security", + "order": 50, + }, + { + "key": "timeoutSeconds", + "label": "Request Timeout (s)", + "type": "number", + "default": 30, + "min": 5, + "max": 300, + "help": "HTTP request timeout in seconds", + "required": False, + "category": "connection", + "order": 51, + }, + { + "key": "showModelSignature", + "label": "Show Model Signature", + "type": "boolean", + "default": False, + "help": "Append a model signature to bot responses", + "required": False, + "category": "output", + "order": 42, + }, + { + "key": "responsePrefix", + "label": "Response Prefix", + "type": "text", + "default": "", + "help": "Optional prefix added before bot responses", + "required": False, + "category": "output", + "order": 43, + }, + { + "key": "commands.native", + "label": "Enable Native Commands", + "type": "boolean", + "default": False, + "help": "Allow slash commands that invoke native Urbit operations", + "required": False, + "category": "commands", + "order": 60, + }, + { + "key": "commands.nativeSkills", + "label": "Native Command Skills", + "type": "list", + "help": "List of native skill names that the bot can invoke via slash commands", + "required": False, + "category": "commands", + "order": 61, + }, + { + "key": "trustedLocalFileRoots", + "label": "Trusted Local File Roots", + "type": "list", + "help": "Trusted local directory paths for file operations (security-sensitive)", + "required": False, + "category": "security", + "order": 52, + }, + { + "key": "authorization.channel_rules", + "label": "Channel Rules", + "type": "object", + "help": 'Per-channel authorization rules (e.g. {"chat/my-group": {"allow": ["zod", "bus"]}})', + "required": False, + "category": "authorization", + "order": 8, + }, + { + "key": "authorization.default_authorized_ships", + "label": "Default Authorized Ships", + "type": "list", + "help": "Ships authorized across all channels by default", + "required": False, + "category": "authorization", + "order": 9, + }, + { + "key": "configWrites", + "label": "Config Writes", + "type": "object", + "default": {}, + "help": "Custom config writes to push to Settings Store on connect", + "required": False, + "category": "settings", + "order": 70, + }, + { + "key": "accounts", + "label": "Multi Accounts", + "type": "list", + "help": "Additional account configurations for multi-ship support", + "required": False, + "category": "connection", + "order": 80, + }, + ] + + +_CATEGORY_LABELS: dict[str, str] = { + "connection": "连接设置", + "authorization": "授权与安全", + "channels": "频道管理", + "invites": "邀请管理", + "performance": "性能与重试", + "threads": "线程管理", + "output": "输出控制", + "security": "安全设置", + "commands": "命令设置", + "settings": "Settings Store", +} + + +def get_categories() -> dict[str, str]: + return dict(_CATEGORY_LABELS) + + +def get_hints_by_category() -> dict[str, list[dict[str, Any]]]: + result: dict[str, list[dict[str, Any]]] = {} + for hint in get_config_ui_hints(): + cat = hint.get("category", "other") + result.setdefault(cat, []).append(hint) + return result + + +def count_hints() -> int: + return len(get_config_ui_hints()) diff --git a/backend/package/yuxi/channels/adapters/urbit/directory.py b/backend/package/yuxi/channels/adapters/urbit/directory.py new file mode 100644 index 00000000..14722e8d --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/directory.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + +from yuxi.utils.logging_config import logger + +if TYPE_CHECKING: + from .client import UrbitClient + + +class DirectoryQuery: + def __init__(self, client: UrbitClient): + self._client = client + + async def list_contacts(self) -> list[dict[str, Any]]: + from .scry import scry_contacts + + try: + data = await scry_contacts(self._client) + if not data or not isinstance(data, dict): + return [] + + contacts: list[dict[str, Any]] = [] + for ship, info in data.items(): + if isinstance(info, dict): + contacts.append( + { + "ship": f"~{ship.lstrip('~')}", + "nickname": info.get("nickname", ""), + "bio": info.get("bio", ""), + "status": info.get("status", ""), + "avatar": info.get("avatar", ""), + "color": info.get("color", ""), + } + ) + + logger.info(f"[Urbit] Directory: found {len(contacts)} contacts") + return contacts + + except Exception as e: + logger.warning(f"[Urbit] Directory query failed: {e}") + return [] + + async def list_peers(self) -> list[str]: + try: + r = await self._client.get("/~/peers", timeout=10.0) + if r.status_code != 200: + logger.warning(f"[Urbit] Directory: /~/peers returned {r.status_code}") + return [] + data = r.json() + peers = [] + for peer_key in data: + if isinstance(peer_key, str): + peers.append(f"~{peer_key.lstrip('~')}") + logger.info(f"[Urbit] Directory: found {len(peers)} peers") + return peers + except Exception as e: + logger.warning(f"[Urbit] Directory peers query failed: {e}") + return [] + + async def search_contact(self, query: str) -> list[dict[str, Any]]: + contacts = await self.list_contacts() + query_lower = query.lower() + return [ + c + for c in contacts + if query_lower in c.get("ship", "").lower() or query_lower in c.get("nickname", "").lower() + ] diff --git a/backend/package/yuxi/channels/adapters/urbit/discovery.py b/backend/package/yuxi/channels/adapters/urbit/discovery.py new file mode 100644 index 00000000..3caeebcc --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/discovery.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING + +from yuxi.utils.logging_config import logger + +from .scry import scry_groups_init + +if TYPE_CHECKING: + from .client import UrbitClient + from .settings import SettingsStore + +_DISCOVERY_INTERVAL = 120.0 + + +class ChannelDiscovery: + def __init__( + self, + client: UrbitClient, + auto_discover: bool = False, + group_channels: list[str] | None = None, + settings_store: SettingsStore | None = None, + ): + self._client = client + self._auto_discover = auto_discover + self._group_channels = set(group_channels or []) + self._discovered_channels: set[str] = set() + self._refresh_task: asyncio.Task | None = None + self._on_new_channel: asyncio.Event | None = None + self._settings_store = settings_store + + @property + def channels(self) -> set[str]: + return self._discovered_channels | self._group_channels + + async def start(self) -> set[str]: + if self._group_channels: + self._discovered_channels = set(self._group_channels) + + if self._settings_store: + store_group_channels = self._settings_store.get("groupChannels", []) + if isinstance(store_group_channels, list) and store_group_channels: + self._discovered_channels |= set(store_group_channels) + logger.info(f"[Urbit] Merged {len(store_group_channels)} channels from Settings Store") + + if self._auto_discover: + discovered = await self.discover_channels() + self._discovered_channels |= discovered + self._refresh_task = asyncio.create_task(self._refresh_loop()) + + channels = self.channels + logger.info(f"[Urbit] Channel discovery started: {len(channels)} channels") + return channels + + async def stop(self) -> None: + if self._refresh_task and not self._refresh_task.done(): + self._refresh_task.cancel() + try: + await self._refresh_task + except asyncio.CancelledError: + pass + self._refresh_task = None + logger.info("[Urbit] Channel discovery stopped") + + async def discover_channels(self) -> set[str]: + discovered: set[str] = set() + try: + init_data = await scry_groups_init(self._client) + if init_data and isinstance(init_data, dict): + groups = init_data.get("groups", {}) + for group_id, group_data in groups.items(): + channels = group_data.get("channels", {}) + for channel_nest in channels: + if channel_nest.startswith("chat/"): + discovered.add(channel_nest) + except Exception as e: + logger.warning(f"[Urbit] Channel discovery failed: {e}") + + if discovered: + logger.info(f"[Urbit] Discovered {len(discovered)} chat channels") + return discovered + + async def _refresh_loop(self) -> None: + while True: + try: + await asyncio.sleep(_DISCOVERY_INTERVAL) + new_channels = await self.discover_channels() + added = new_channels - self._discovered_channels + if added: + self._discovered_channels |= added + logger.info(f"[Urbit] Discovered {len(added)} new channels: {added}") + except asyncio.CancelledError: + break + except Exception as e: + logger.warning(f"[Urbit] Channel refresh error: {e}") + + async def persist_group_channels(self) -> None: + if not self._settings_store: + return + try: + store = self._settings_store.get_all() + chat_channels = [ch for ch in self.channels if ch.startswith("chat/")] + store["groupChannels"] = chat_channels + self._settings_store._store = store + logger.info(f"[Urbit] Persisted {len(chat_channels)} group channels to Settings Store") + except Exception as e: + logger.warning(f"[Urbit] Failed to persist group channels: {e}") + + def add_channel(self, channel_nest: str) -> None: + if channel_nest not in self._discovered_channels: + self._discovered_channels.add(channel_nest) + logger.info(f"[Urbit] Manually added channel: {channel_nest}") + + def get_chat_nests(self, ship_name: str) -> list[str]: + nests: list[str] = [] + for channel in self.channels: + if channel.startswith("chat/"): + group_name = channel[len("chat/") :] + nests.append(f"~{ship_name}/{group_name}/chat") + return nests diff --git a/backend/package/yuxi/channels/adapters/urbit/doctor.py b/backend/package/yuxi/channels/adapters/urbit/doctor.py new file mode 100644 index 00000000..e6eaca1d --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/doctor.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from typing import Any + +from yuxi.utils.logging_config import logger + + +_LEGACY_KEY_MAP: dict[str, str] = { + "allow_private_network": "network.dangerously_allow_private_network", + "allowPrivateNetwork": "network.dangerouslyAllowPrivateNetwork", +} + + +def create_legacy_private_network_doctor_contract( + config: dict[str, Any], +) -> list[dict[str, str]]: + migrations: list[dict[str, str]] = [] + for legacy_key, new_key in _LEGACY_KEY_MAP.items(): + if legacy_key in config: + migrations.append( + { + "from": legacy_key, + "to": new_key, + "value": config[legacy_key], + } + ) + return migrations + + +def apply_doctor_migrations( + config: dict[str, Any], + migrations: list[dict[str, str]], +) -> dict[str, Any]: + config = dict(config) + for mig in migrations: + legacy_key = mig["from"] + new_key = mig["to"] + value = mig["value"] + + config.setdefault("network", {}) + config["network"]["dangerously_allow_private_network"] = value + config["network"]["dangerouslyAllowPrivateNetwork"] = value + + config.pop(legacy_key, None) + logger.info(f"[Urbit] Doctor: migrated {legacy_key} → {new_key} = {value}") + + return config + + +def is_legacy_private_network_config(config: dict[str, Any]) -> bool: + return "allow_private_network" in config or "allowPrivateNetwork" in config diff --git a/backend/package/yuxi/channels/adapters/urbit/dual_plugin.py b/backend/package/yuxi/channels/adapters/urbit/dual_plugin.py new file mode 100644 index 00000000..cda3d1fd --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/dual_plugin.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + +from yuxi.utils.logging_config import logger + +if TYPE_CHECKING: + from .adapter import UrbitAdapter + + +class UrbitSetupPlugin: + def __init__(self, config: dict[str, Any] | None = None): + self._config = config or {} + self._legacy_migrations_detected = False + + @property + def config(self) -> dict[str, Any]: + return self._config + + def detect_legacy_state_migrations(self) -> list[dict[str, Any]]: + from .doctor import create_legacy_private_network_doctor_contract + + migrations = create_legacy_private_network_doctor_contract(self._config) + if migrations: + self._legacy_migrations_detected = True + logger.info(f"[Urbit SetupPlugin] Detected {len(migrations)} legacy config migrations") + return migrations + + def check_ship_url_reachability(self) -> dict[str, Any]: + ship_url = self._config.get("ship_url", "") + if not ship_url: + return {"reachable": False, "error": "No ship_url configured"} + + from .probe import validate_urbit_url + + is_valid, warnings = validate_urbit_url(ship_url) + return { + "reachable": is_valid, + "url": ship_url, + "warnings": warnings, + } + + def validate_setup(self) -> list[str]: + errors: list[str] = [] + + ship_url = self._config.get("ship_url", "") + if not ship_url: + errors.append("ship_url is missing") + elif not ship_url.startswith(("http://", "https://")): + errors.append("ship_url must use http:// or https:// protocol") + + ship_name = self._config.get("ship_name", "") + if not ship_name: + errors.append("ship_name is missing") + + ship_code = self._config.get("ship_code", "") + if not ship_code: + errors.append("ship_code is missing") + + dm_policy = self._config.get("dm_policy", "pairing") + if dm_policy not in ("open", "disabled", "pairing", "allowlist"): + errors.append(f"dm_policy '{dm_policy}' is invalid") + + group_policy = self._config.get("group_policy", "open") + if group_policy not in ("open", "disabled", "allowlist"): + errors.append(f"group_policy '{group_policy}' is invalid") + + return errors + + def get_setup_schema(self) -> dict[str, Any]: + from .setup import get_setup_schema + + return get_setup_schema() + + def get_config_ui_hints(self) -> list[dict[str, Any]]: + from .config_ui_hints import get_config_ui_hints + + return get_config_ui_hints() + + @property + def has_legacy_migrations(self) -> bool: + return self._legacy_migrations_detected + + +class UrbitAccountPlugin: + def __init__(self, adapter: UrbitAdapter): + self._adapter = adapter + + @property + def adapter(self) -> UrbitAdapter: + return self._adapter + + async def handle_inbound(self, raw: dict[str, Any]) -> Any: + return self._adapter.normalize_inbound(raw) + + def format_outbound(self, response) -> dict[str, Any]: + return self._adapter.format_outbound(response) + + def build_stream_identity(self, chat_id: str, msg_id: str): + return self._adapter._build_stream_identity(chat_id, msg_id) + + async def fetch_thread_history(self, resource_path: str, parent_id: str) -> list[dict[str, Any]]: + return await self._adapter.fetch_thread_history(resource_path, parent_id) + + def build_thread_context(self, history: list[dict[str, Any]]) -> str: + return self._adapter.build_thread_context(history) + + async def get_user_info(self, channel_user_id: str) -> dict[str, Any]: + return await self._adapter.get_user_info(channel_user_id) + + async def download_media(self, file_id: str) -> bytes: + return await self._adapter.download_media(file_id) + + +class DualPluginRegistry: + def __init__(self): + self._setup_plugin: UrbitSetupPlugin | None = None + self._account_plugins: dict[str, UrbitAccountPlugin] = {} + + def register_setup_plugin(self, plugin: UrbitSetupPlugin) -> None: + self._setup_plugin = plugin + logger.debug("[Urbit] SetupPlugin registered") + + def register_account_plugin(self, name: str, plugin: UrbitAccountPlugin) -> None: + self._account_plugins[name] = plugin + logger.debug(f"[Urbit] AccountPlugin '{name}' registered") + + def unregister_account_plugin(self, name: str) -> None: + self._account_plugins.pop(name, None) + logger.debug(f"[Urbit] AccountPlugin '{name}' unregistered") + + def get_setup_plugin(self) -> UrbitSetupPlugin | None: + return self._setup_plugin + + def get_account_plugin(self, name: str = "default") -> UrbitAccountPlugin | None: + return self._account_plugins.get(name) + + def list_account_plugins(self) -> list[str]: + return list(self._account_plugins.keys()) + + def clear(self) -> None: + self._setup_plugin = None + self._account_plugins.clear() + logger.debug("[Urbit] DualPluginRegistry cleared") + + +_global_registry: DualPluginRegistry | None = None + + +def get_global_registry() -> DualPluginRegistry: + global _global_registry + if _global_registry is None: + _global_registry = DualPluginRegistry() + return _global_registry diff --git a/backend/package/yuxi/channels/adapters/urbit/errors.py b/backend/package/yuxi/channels/adapters/urbit/errors.py new file mode 100644 index 00000000..e11f8e9e --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/errors.py @@ -0,0 +1,19 @@ +from __future__ import annotations + + +class UrbitError(Exception): + pass + + +class UrbitUrlError(UrbitError): + pass + + +class UrbitHttpError(UrbitError): + def __init__(self, status_code: int, message: str = ""): + self.status_code = status_code + super().__init__(message or f"Urbit HTTP {status_code}") + + +class UrbitAuthError(UrbitError): + pass diff --git a/backend/package/yuxi/channels/adapters/urbit/format.py b/backend/package/yuxi/channels/adapters/urbit/format.py new file mode 100644 index 00000000..8caa6a3e --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/format.py @@ -0,0 +1,259 @@ +from __future__ import annotations + +import re +from typing import Any + +_SHIP_PATTERN = re.compile(r"^~[a-z]{3,}(-[a-z]{3,})*$", re.IGNORECASE) + + +def is_valid_ship(ship: str) -> bool: + return bool(_SHIP_PATTERN.match(ship)) + + +def strip_ship_tilde(ship: str) -> str: + return ship.lstrip("~") + + +def ensure_ship_tilde(ship: str) -> str: + return ship if ship.startswith("~") else f"~{ship}" + + +def build_dm_channel_id(target_ship: str, sender_ship: str) -> str: + t = strip_ship_tilde(target_ship) + s = strip_ship_tilde(sender_ship) + return f"~{t}/dm--{s}" + + +def build_group_channel_id(host_ship: str, group_name: str, channel_type: str = "chat") -> str: + h = strip_ship_tilde(host_ship) + return f"~{h}/{group_name}/{channel_type}" + + +def build_channel_path(chat_type: str, **kwargs) -> str: + if chat_type == "direct": + target = strip_ship_tilde(kwargs["target_ship"]) + return f"/dm/~{target}" + group_name = kwargs["group_name"] + ch_type = kwargs.get("channel_type", "chat") + return f"/{ch_type}/{group_name}" + + +def parse_graph_update(raw: dict[str, Any]) -> dict[str, Any]: + graph_update = raw.get("graph-update", {}) + if not isinstance(graph_update, dict) or not graph_update: + return {} + + result: dict[str, Any] = {} + resource = graph_update.get("resource", {}) + result["resource_path"] = resource.get("path", "") + result["resource_type"] = resource.get("type", "chat") + result["ship"] = graph_update.get("ship", "") + result["index"] = graph_update.get("index", "") + result["time"] = graph_update.get("time") + + additions = graph_update.get("additions", {}) + if additions: + result["content"] = _extract_graph_content(additions) + + if "reaction" in graph_update: + result["reaction"] = graph_update["reaction"] + + return result + + +def _extract_graph_content(additions: dict[str, Any]) -> str: + parts: list[str] = [] + for node_data in additions.values(): + post = node_data.get("post", {}) + contents = post.get("contents", []) + for item in contents: + if "text" in item: + parts.append(item["text"]) + elif "mention" in item: + parts.append(f"~{item['mention']}") + elif "url" in item: + parts.append(item["url"]) + elif "code" in item: + code_block = item["code"] + lang = code_block.get("lang", "") + code_text = code_block.get("code", "") if isinstance(code_block, dict) else str(code_block) + if lang: + parts.append(f"```{lang}\n{code_text}\n```\n") + else: + parts.append(f"```\n{code_text}\n```\n") + elif "blockquote" in item: + quoted = item["blockquote"] + if isinstance(quoted, list): + quoted_text = _extract_content_list(quoted) + parts.append(f"> {quoted_text}\n") + else: + parts.append(f"> {quoted}\n") + elif "image" in item: + img = item["image"] + img_url = img.get("src", "") if isinstance(img, dict) else str(img) + if img_url: + parts.append(f"[Image: {img_url}]") + elif "ship" in item: + parts.append(f"~{item['ship']}") + elif isinstance(item, str): + parts.append(item) + return "".join(parts) + + +def _extract_content_list(contents: list[Any]) -> str: + parts: list[str] = [] + for item in contents: + if isinstance(item, dict): + if "text" in item: + parts.append(item["text"]) + elif "mention" in item: + parts.append(f"~{item['mention']}") + elif "url" in item: + parts.append(item["url"]) + elif "code" in item: + code_block = item["code"] + code_text = code_block.get("code", "") if isinstance(code_block, dict) else str(code_block) + parts.append(code_text) + elif "ship" in item: + parts.append(f"~{item['ship']}") + elif isinstance(item, str): + parts.append(item) + return "".join(parts) + + +def extract_graph_mentions(additions: dict[str, Any]) -> list[str]: + mentions: list[str] = [] + for node_data in additions.values(): + post = node_data.get("post", {}) + contents = post.get("contents", []) + for item in contents: + if "mention" in item: + mentions.append(item["mention"]) + return mentions + + +def extract_graph_attachments(additions: dict[str, Any]) -> list[dict[str, Any]]: + from yuxi.channels.models import Attachment + + attachments: list[dict[str, Any]] = [] + for node_data in additions.values(): + post = node_data.get("post", {}) + contents = post.get("contents", []) + for item in contents: + if "image" in item: + img = item["image"] + src = img.get("src", "") if isinstance(img, dict) else str(img) + if src: + attachment = Attachment( + type="image", + url=src, + mime_type="image/*", + ) + attachments.append(attachment.model_dump()) + elif "file" in item: + file_info = item["file"] + if isinstance(file_info, dict): + attachment = Attachment( + type="file", + url=file_info.get("src", ""), + filename=file_info.get("name"), + size_bytes=file_info.get("size"), + ) + attachments.append(attachment.model_dump()) + return attachments + + +def build_poke_payload( + host_ship: str, + content: str, + channel_path: str, + *, + continuation: bool = False, + reply_to: str | None = None, + media_content: list[dict[str, Any]] | None = None, + chat_type: str = "group", +) -> dict[str, Any]: + _MAX_CONTENT_LENGTH = 100_000 + if len(content) > _MAX_CONTENT_LENGTH: + raise ValueError(f"Content length {len(content)} exceeds maximum {_MAX_CONTENT_LENGTH}") + + mark = "chat-dm-action" if chat_type == "direct" else "chat-message" + + if media_content: + content_list = media_content + else: + from .story import markdown_to_story + + story = markdown_to_story(content) + content_list = story + + payload: dict[str, Any] = { + "action": "poke", + "ship": host_ship, + "app": "chat", + "mark": mark, + "json": { + "path": channel_path, + "envelope": { + "content": content_list, + "author": f"~{host_ship}", + }, + }, + } + if continuation: + payload["json"]["envelope"]["continuation"] = True + if reply_to: + payload["json"]["envelope"]["reply_to"] = reply_to + return payload + + +def build_chat_action_payload( + host_ship: str, + channel_path: str, + action: str, + msg_id: str, + *, + content: str | None = None, + emoji: str | None = None, +) -> dict[str, Any]: + action_data: dict[str, Any] = {} + match action: + case "add-react": + action_data = {"add-react": {"react": emoji, "msg": msg_id}} + case "del-react": + action_data = {"del-react": {"react": emoji, "msg": msg_id}} + case "edit": + action_data = {"edit": {"msg": msg_id, "content": [{"text": content}]}} + case "del": + action_data = {"del": {"msg": msg_id}} + case "typing": + action_data = {"typing": None} + case _: + raise ValueError(f"Unknown chat action: {action}") + + return { + "action": "poke", + "ship": host_ship, + "app": "chat", + "mark": "chat-action", + "json": { + "path": channel_path, + "action": action_data, + }, + } + + +def build_media_content(media_type: str, data: str) -> list[dict[str, Any]]: + match media_type: + case "image": + return [{"image": {"src": data, "width": 0, "height": 0}}] + case "video": + return [{"video": {"src": data}}] + case "audio": + return [{"audio": {"src": data}}] + case "url": + return [{"url": data}] + case "reference": + return [{"reference": {"text": data}}] + case _: + return [{"text": data}] diff --git a/backend/package/yuxi/channels/adapters/urbit/history.py b/backend/package/yuxi/channels/adapters/urbit/history.py new file mode 100644 index 00000000..e9e4001f --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/history.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import asyncio +from collections import OrderedDict +from typing import Any, TYPE_CHECKING + + +if TYPE_CHECKING: + pass + +_MAX_CACHE_SIZE = 100 + + +class MessageCache: + def __init__(self): + self._cache: dict[str, OrderedDict[str, dict[str, Any]]] = {} + self._sent_ids: OrderedDict[str, float] = OrderedDict() + self._sent_max = 200 + self._lock = asyncio.Lock() + + async def cache_message(self, msg_id: str, channel_id: str, content: str, author: str) -> None: + entry = { + "id": msg_id, + "author": author, + "content": content, + } + async with self._lock: + if channel_id not in self._cache: + self._cache[channel_id] = OrderedDict() + + if msg_id: + self._cache[channel_id][msg_id] = entry + self._cache[channel_id].move_to_end(msg_id) + + if len(self._cache[channel_id]) > _MAX_CACHE_SIZE: + self._cache[channel_id].popitem(last=False) + + async def get_channel_history(self, channel_id: str, limit: int = 50) -> list[dict[str, Any]]: + async with self._lock: + if channel_id not in self._cache: + return [] + items = list(self._cache[channel_id].values()) + return items[-limit:] + + def get_recent(self, channel_id: str, limit: int = 50) -> list[dict[str, Any]]: + if channel_id not in self._cache: + return [] + items = list(self._cache[channel_id].values()) + return items[-limit:] + + async def track_sent_message(self, msg_id: str) -> None: + import time + + async with self._lock: + self._sent_ids[msg_id] = time.monotonic() + self._sent_ids.move_to_end(msg_id) + if len(self._sent_ids) > self._sent_max: + self._sent_ids.popitem(last=False) + + async def is_sent(self, msg_id: str) -> bool: + async with self._lock: + return msg_id in self._sent_ids + + async def clear_channel(self, channel_id: str) -> None: + async with self._lock: + self._cache.pop(channel_id, None) diff --git a/backend/package/yuxi/channels/adapters/urbit/invites.py b/backend/package/yuxi/channels/adapters/urbit/invites.py new file mode 100644 index 00000000..4f3997d5 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/invites.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from typing import Any + +from yuxi.utils.logging_config import logger + + +class InviteManager: + def __init__( + self, + auto_accept_groups: bool = False, + group_invite_allowlist: list[str] | None = None, + auto_accept_dm: bool = False, + dm_allowlist: list[str] | None = None, + ): + self.auto_accept_groups = auto_accept_groups + self.group_invite_allowlist = [s.lstrip("~").lower() for s in (group_invite_allowlist or [])] + self.auto_accept_dm = auto_accept_dm + self.dm_allowlist = [s.lstrip("~").lower() for s in (dm_allowlist or [])] + self._pending_invites: dict[str, dict[str, Any]] = {} + + def should_auto_accept_group_invite(self, inviter_ship: str) -> bool: + if not self.auto_accept_groups: + return False + ship = inviter_ship.lstrip("~").lower() + if not self.group_invite_allowlist: + return False + return ship in self.group_invite_allowlist + + def should_auto_accept_dm(self, inviter_ship: str) -> bool: + if not self.auto_accept_dm: + return False + ship = inviter_ship.lstrip("~").lower() + if not self.dm_allowlist: + return False + return ship in self.dm_allowlist + + async def handle_foreign_update(self, foreign_update: dict[str, Any]) -> dict[str, Any] | None: + group = foreign_update.get("group", "") + ship = foreign_update.get("ship", "").lstrip("~") + join = foreign_update.get("join", False) + + if not group or not ship: + return None + + invite_id = f"{group}:{ship}" + + if join: + self._pending_invites[invite_id] = foreign_update + + if self.should_auto_accept_group_invite(ship): + logger.info(f"[Urbit] Auto-accepting group invite from ~{ship} for {group}") + await self._accept_invite(group, ship) + self._pending_invites.pop(invite_id, None) + return {"action": "auto_accepted", "group": group, "ship": ship} + + logger.info(f"[Urbit] Group invite pending: {group} from ~{ship}") + return {"action": "pending", "group": group, "ship": ship, "invite_id": invite_id} + + if not join and invite_id in self._pending_invites: + self._pending_invites.pop(invite_id, None) + logger.info(f"[Urbit] Ship ~{ship} left {group}, invite resolved") + + return {"action": "left", "group": group, "ship": ship} + + async def _accept_invite(self, group: str, inviter: str) -> None: + logger.info(f"[Urbit] Would accept group invite: {group} from ~{inviter}") + + def get_pending_invites(self) -> list[dict[str, Any]]: + return list(self._pending_invites.values()) diff --git a/backend/package/yuxi/channels/adapters/urbit/media.py b/backend/package/yuxi/channels/adapters/urbit/media.py new file mode 100644 index 00000000..5a4cb9f4 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/media.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from yuxi.utils.logging_config import logger + +if TYPE_CHECKING: + from .client import UrbitClient + + +async def download_media( + client: UrbitClient, + file_url: str, + timeout: float = 30.0, + *, + allow_private: bool = False, +) -> tuple[bytes, str]: + from .probe import validate_urbit_url + + is_valid, warnings = validate_urbit_url(file_url, allow_private=allow_private) + if not is_valid: + raise ValueError(f"SSRF blocked: {'; '.join(warnings)}") + + import httpx + + try: + r = await client.http.get( + file_url, + timeout=httpx.Timeout(timeout), + follow_redirects=True, + ) + r.raise_for_status() + content_type = r.headers.get("content-type", "application/octet-stream") + return r.content, content_type + except Exception as e: + logger.warning(f"[Urbit] Media download failed from {file_url}: {e}") + raise + + +_MEDIA_EXTENSIONS: dict[str, str] = { + "image/jpeg": ".jpg", + "image/png": ".png", + "image/gif": ".gif", + "image/webp": ".webp", + "image/svg+xml": ".svg", + "image/heic": ".heic", + "image/heif": ".heif", + "video/mp4": ".mp4", + "video/webm": ".webm", + "audio/mpeg": ".mp3", + "audio/ogg": ".ogg", + "audio/wav": ".wav", +} + + +def get_media_extension(content_type: str) -> str: + return _MEDIA_EXTENSIONS.get(content_type, ".bin") diff --git a/backend/package/yuxi/channels/adapters/urbit/monitor.py b/backend/package/yuxi/channels/adapters/urbit/monitor.py new file mode 100644 index 00000000..9abea298 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/monitor.py @@ -0,0 +1,712 @@ +from __future__ import annotations + +import asyncio +import json +import random +import re +import time +from collections import OrderedDict +from datetime import datetime, UTC +from typing import Any, TYPE_CHECKING +from collections.abc import Callable + +import httpx + +from yuxi.channels.exceptions import MessageFormatError +from yuxi.channels.models import ( + ChannelIdentity, + ChannelMessage, + ChannelType, + ChatType, + MentionsInfo, +) +from yuxi.utils.logging_config import logger + +from .format import ( + parse_graph_update, + ensure_ship_tilde, + extract_graph_mentions, + extract_graph_attachments, +) +from .cites import parse_cite + +if TYPE_CHECKING: + from .client import UrbitClient + from .threads import ThreadManager + from .history import MessageCache + +_MAX_RECONNECT_DELAY = 30.0 +_INITIAL_RECONNECT_DELAY = 1.0 +_MAX_RECONNECT_COUNT = 10 +_JITTER = 0.1 +_ACK_INTERVAL = 20 +_ALL_PATTERN = re.compile(r"\b@(all|everyone|channel|here)\b", re.IGNORECASE) + + +class SSEDeduplicator: + def __init__(self, max_size: int = 2000): + self._seen: OrderedDict[str, float] = OrderedDict() + self._max_size = max_size + self._lock = asyncio.Lock() + + async def is_duplicate(self, graph_index: str) -> bool: + async with self._lock: + now = time.monotonic() + if graph_index in self._seen: + if now - self._seen[graph_index] < 60: + return True + + self._seen[graph_index] = now + self._seen.move_to_end(graph_index) + + if len(self._seen) > self._max_size: + self._seen.popitem(last=False) + + return False + + +class UrbitSSEManager: + """多订阅 SSE 管理器，负责 SSE 连接的创建、ACK 和优雅关闭""" + + def __init__( + self, + client: UrbitClient, + ship_url: str, + on_message: Callable[[ChannelMessage], Any], + status_check: Callable[[], bool], + reconnect_delay: float = 5.0, + dedup: SSEDeduplicator | None = None, + bot_ship_name: str = "", + reauth_callback: Callable[[], Any] | None = None, + thread_manager: ThreadManager | None = None, + message_cache: MessageCache | None = None, + ): + self._client = client + self._ship_url = ship_url + self._on_message = on_message + self._status_check = status_check + self._reconnect_delay = reconnect_delay + self._dedup = dedup or SSEDeduplicator() + self._bot_ship_name = bot_ship_name.lstrip("~") + self._subscriptions: dict[str, asyncio.Task | None] = {} + self._channel_ids: dict[str, str] = {} + self._event_counts: dict[str, int] = {} + self._run_lock = asyncio.Lock() + self._reauth_callback = reauth_callback + self._thread_manager = thread_manager + self._message_cache = message_cache + self._contacts: dict[str, str] = {} + + async def start(self, subscriptions: dict[str, str] | None = None) -> None: + """启动 SSE 订阅管理器 + + subscriptions: {label: sse_path}，默认监听 chat-store + """ + if subscriptions is None: + subscriptions = {"chat": "/~/channel/chat-store"} + + for label, sse_path in subscriptions.items(): + channel_id = f"{int(time.time() * 1000)}-{label}" + self._channel_ids[label] = channel_id + self._event_counts[label] = 0 + + task = asyncio.create_task(self._run_subscription(label, sse_path, channel_id)) + self._subscriptions[label] = task + logger.info(f"[Urbit] SSE subscription '{label}' → {sse_path} (channel: {channel_id})") + + async def stop(self) -> None: + for label, task in list(self._subscriptions.items()): + if task and not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + await self._unsubscribe_all() + + for label, channel_id in list(self._channel_ids.items()): + try: + await self._unsubscribe_channel(channel_id) + except Exception as e: + logger.warning(f"[Urbit] Failed to unsubscribe channel '{label}': {e}") + + await self._release_streams() + + self._subscriptions.clear() + self._channel_ids.clear() + self._event_counts.clear() + logger.info("[Urbit] All SSE subscriptions stopped") + + async def _unsubscribe_all(self) -> None: + task_count = 0 + for label, task in list(self._subscriptions.items()): + if task and not task.done(): + task.cancel() + task_count += 1 + if task_count > 0: + await asyncio.sleep(0.1) + logger.debug(f"[Urbit] Unsubscribed all {task_count} SSE tasks") + + async def _release_streams(self) -> None: + try: + await self._client.http.aclose() + except Exception: + pass + logger.debug("[Urbit] SSE streams released") + + async def _unsubscribe_channel(self, channel_id: str) -> None: + try: + r = await self._client.delete(f"/~/channel/{channel_id}") + if r.status_code not in (200, 204): + logger.debug(f"[Urbit] Channel DELETE {channel_id} → {r.status_code}") + except Exception as e: + logger.debug(f"[Urbit] Channel DELETE {channel_id} failed: {e}") + + async def _send_ack(self, channel_id: str) -> None: + try: + await self._client.put( + f"/~/channel/{channel_id}", + json={"action": "ack"}, + timeout=5.0, + ) + except Exception: + pass + + async def _run_subscription(self, label: str, sse_path: str, channel_id: str) -> None: + delay = _INITIAL_RECONNECT_DELAY + reconnect_count = 0 + + while self._status_check(): + try: + response = await self._client.stream_get(sse_path) + response.raise_for_status() + + logger.info(f"[Urbit] SSE '{label}' connected to {self._ship_url}{sse_path}") + delay = _INITIAL_RECONNECT_DELAY + reconnect_count = 0 + self._event_counts[label] = 0 + + async for line in response.aiter_lines(): + if not self._status_check(): + break + + if not line or not line.startswith("data:"): + continue + + data_str = line[5:].strip() + if not data_str: + continue + + try: + data = json.loads(data_str) + except json.JSONDecodeError: + continue + + label_handlers = { + "chat": lambda d: self._process_chat_event(d, channel_id, label), + "channels": lambda d: self._process_channels_event(d, channel_id, label), + "contacts": lambda d: self._process_contacts_event(d, label), + "foreigns": lambda d: self._process_groups_event(d, label), + "settings": lambda d: self._process_settings_event(d, label), + "groups-ui": lambda d: self._process_groups_ui_event(d, label), + } + handler = label_handlers.get(label) + if handler: + await handler(data) + else: + await self._process_generic_event(data, label) + + except httpx.RequestError as e: + logger.warning(f"[Urbit] SSE '{label}' connection lost: {e}. Reconnecting in {delay:.1f}s...") + except Exception as e: + logger.error(f"[Urbit] SSE '{label}' listener error: {e}") + + reconnect_count += 1 + if reconnect_count > _MAX_RECONNECT_COUNT: + logger.error( + f"[Urbit] SSE '{label}' exceeded max reconnect count ({_MAX_RECONNECT_COUNT}), pausing 10s..." + ) + await asyncio.sleep(10.0) + reconnect_count = 0 + delay = _INITIAL_RECONNECT_DELAY + + if self._reauth_callback: + try: + await self._reauth_callback() + logger.info(f"[Urbit] SSE '{label}' re-authenticated before reconnect") + except Exception as e: + logger.warning(f"[Urbit] SSE '{label}' re-auth failed: {e}") + + jitter_ms = delay * _JITTER * random.random() + await asyncio.sleep(delay + jitter_ms) + delay = min(delay * 2, _MAX_RECONNECT_DELAY) + + async def _process_chat_event(self, data: dict[str, Any], channel_id: str, label: str) -> None: + graph_update = data.get("graph-update") + if not graph_update: + return + + edit = graph_update.get("edit") + if edit and isinstance(edit, dict): + self._handle_edit_event(edit, channel_id, label) + return + + index = str(graph_update.get("index", "")) + if index and await self._dedup.is_duplicate(index): + return + + self._event_counts[label] = self._event_counts.get(label, 0) + 1 + if self._event_counts[label] % _ACK_INTERVAL == 0: + asyncio.create_task(self._send_ack(channel_id)) + + try: + msg = _graph_update_to_channel_message(data, self._client.ship_name) + if not msg.content: + return + + if _is_self_message(msg, self._bot_ship_name): + return + + if msg.attachments: + attachment_hints = [] + for att in msg.attachments: + url = att.get("url", "") + content_type = att.get("mime_type", "application/octet-stream") + attachment_hints.append(f"[media attached: {url} ({content_type})]") + if attachment_hints: + hint_text = " ".join(attachment_hints) + msg.content = f"{hint_text}\n{msg.content}" + + if self._thread_manager and self._thread_manager.is_thread_reply(data): + parent_id = self._thread_manager.get_reply_parent_id(data) + if parent_id: + msg.metadata["thread_parent_id"] = parent_id + thread_key = f"{msg.identity.channel_chat_id}:{parent_id}" + if not self._thread_manager.has_participated(thread_key): + mentions = _detect_bot_mentions(msg, self._bot_ship_name) + if not mentions: + logger.debug( + f"[Urbit] Thread reply without mention in {msg.identity.channel_chat_id}, " + f"skipping (not participated)" + ) + return + + mentions = _detect_bot_mentions(msg, self._bot_ship_name) + if mentions: + msg.mentions = mentions + if self._thread_manager: + parent_id = msg.metadata.get("thread_parent_id") + if parent_id: + thread_key = f"{msg.identity.channel_chat_id}:{parent_id}" + self._thread_manager.add_participated(thread_key) + + cites_info = _extract_cites_from_message(msg) + if cites_info: + msg.metadata["cites"] = cites_info + + await self._on_message(msg) + + if self._message_cache and msg.metadata.get("msg_id"): + await self._message_cache.cache_message( + msg.metadata["msg_id"], + msg.identity.channel_chat_id, + msg.content, + msg.metadata.get("urbit_ship", ""), + ) + + except MessageFormatError: + logger.warning(f"[Urbit] SSE unparseable event: {json.dumps(data)[:200]}") + except Exception as e: + logger.error(f"[Urbit] SSE event processing error: {e}") + + def _handle_edit_event(self, edit: dict[str, Any], channel_id: str, label: str) -> None: + index = edit.get("index", "") + content_blocks = edit.get("content", []) + extracted = self._extract_edit_content(content_blocks) + if extracted: + logger.info(f"[Urbit] SSE edit event at index={index}: content updated ({len(extracted)} chars)") + + @staticmethod + def _extract_edit_content(contents: list[dict[str, Any]]) -> str: + parts: list[str] = [] + for item in contents: + if isinstance(item, dict) and "text" in item: + parts.append(item["text"]) + return "".join(parts) + + async def _process_generic_event(self, data: dict[str, Any], label: str) -> None: + logger.debug(f"[Urbit] SSE '{label}' event: {json.dumps(data)[:200]}") + + async def _process_channels_event(self, data: dict[str, Any], channel_id: str, label: str) -> None: + add = data.get("add") + if not add or not isinstance(add, dict): + return + + nest = add.get("nest", "") + post = add.get("post", {}) + + if not post: + return + + index = post.get("index", "") + if index and await self._dedup.is_duplicate(f"chan:{nest}:{index}"): + return + + self._event_counts[label] = self._event_counts.get(label, 0) + 1 + if self._event_counts[label] % _ACK_INTERVAL == 0: + asyncio.create_task(self._send_ack(channel_id)) + + graph_data = { + "graph-update": { + "resource": {"type": "chat", "path": nest}, + "ship": post.get("author", ""), + "index": index, + "time": post.get("time-sent"), + "additions": {index: {"post": post}}, + } + } + + try: + msg = _graph_update_to_channel_message(graph_data, self._client.ship_name) + if not msg.content: + return + + if _is_self_message(msg, self._bot_ship_name): + return + + if msg.attachments: + attachment_hints = [] + for att in msg.attachments: + url = att.get("url", "") + content_type = att.get("mime_type", "application/octet-stream") + attachment_hints.append(f"[media attached: {url} ({content_type})]") + if attachment_hints: + hint_text = " ".join(attachment_hints) + msg.content = f"{hint_text}\n{msg.content}" + + if self._thread_manager and self._thread_manager.is_thread_reply(graph_data): + parent_id = self._thread_manager.get_reply_parent_id(graph_data) + if parent_id: + msg.metadata["thread_parent_id"] = parent_id + thread_key = f"{msg.identity.channel_chat_id}:{parent_id}" + if not self._thread_manager.has_participated(thread_key): + mentions = _detect_bot_mentions(msg, self._bot_ship_name) + if not mentions: + logger.debug( + f"[Urbit] Channels thread reply without mention in {msg.identity.channel_chat_id}, " + f"skipping (not participated)" + ) + return + + mentions = _detect_bot_mentions(msg, self._bot_ship_name) + if mentions: + msg.mentions = mentions + if self._thread_manager: + parent_id = msg.metadata.get("thread_parent_id") + if parent_id: + thread_key = f"{msg.identity.channel_chat_id}:{parent_id}" + self._thread_manager.add_participated(thread_key) + + cites_info = _extract_cites_from_message(msg) + if cites_info: + msg.metadata["cites"] = cites_info + + await self._on_message(msg) + + if self._message_cache and msg.metadata.get("msg_id"): + await self._message_cache.cache_message( + msg.metadata["msg_id"], + msg.identity.channel_chat_id, + msg.content, + msg.metadata.get("urbit_ship", ""), + ) + + except MessageFormatError: + logger.warning(f"[Urbit] SSE channels unparseable event: {json.dumps(data)[:200]}") + except Exception as e: + logger.error(f"[Urbit] SSE channels event processing error: {e}") + + async def _process_contacts_event(self, data: dict[str, Any], label: str) -> None: + con = data.get("con") + if not con or not isinstance(con, dict): + return + + who = con.get("who", "") + nick = con.get("nick", "") + if who: + ship = who.lstrip("~") + if nick: + self._contacts[ship] = nick + logger.info(f"[Urbit] Contact update: ~{ship} nick='{nick}'") + else: + self._contacts.pop(ship, None) + logger.info(f"[Urbit] Contact removed: ~{ship}") + + async def _process_groups_event(self, data: dict[str, Any], label: str) -> None: + foreign_update = data.get("foreignUpdate") + if not foreign_update or not isinstance(foreign_update, dict): + return + + group = foreign_update.get("group", "") + ship = foreign_update.get("ship", "") + join = foreign_update.get("join", False) + + if group and ship: + action = "joined" if join else "left" + logger.info(f"[Urbit] Group foreign update: ~{ship} {action} {group}") + if hasattr(self, "_on_foreign_update") and self._on_foreign_update: + await self._on_foreign_update(foreign_update) + + def set_foreign_update_handler(self, handler: Callable[[dict[str, Any]], Any]) -> None: + self._on_foreign_update: Callable[[dict[str, Any]], Any] | None = handler + + async def _process_groups_ui_event(self, data: dict[str, Any], label: str) -> None: + add = data.get("add") + if add and isinstance(add, dict): + group_data = add.get("group", {}) + channels = add.get("channels", []) + group_id = group_data.get("id", "") if isinstance(group_data, dict) else "" + if group_id: + logger.info(f"[Urbit] Groups-UI add: group={group_id}, channels={channels}") + if hasattr(self, "_on_groups_ui_update") and self._on_groups_ui_update: + await self._on_groups_ui_update({"action": "add", "group_id": group_id, "channels": channels}) + return + + join = data.get("join") + if join and isinstance(join, dict): + group_path = join.get("group", "") + ship = join.get("ship", "") + if group_path: + logger.info(f"[Urbit] Groups-UI join: group={group_path}, ship={ship}") + if hasattr(self, "_on_groups_ui_update") and self._on_groups_ui_update: + await self._on_groups_ui_update({"action": "join", "group_path": group_path, "ship": ship}) + return + + kick = data.get("kick") + if kick and isinstance(kick, dict): + group_path = kick.get("group", "") + ship = kick.get("ship", "") + if group_path: + logger.info(f"[Urbit] Groups-UI kick: group={group_path}, ship={ship}") + if hasattr(self, "_on_groups_ui_update") and self._on_groups_ui_update: + await self._on_groups_ui_update({"action": "kick", "group_path": group_path, "ship": ship}) + + def set_groups_ui_update_handler(self, handler: Callable[[dict[str, Any]], Any]) -> None: + self._on_groups_ui_update: Callable[[dict[str, Any]], Any] | None = handler + + @property + def contacts(self) -> dict[str, str]: + return dict(self._contacts) + + async def _process_settings_event(self, data: dict[str, Any], label: str) -> None: + settings_update = data.get("settings-event") + if not settings_update: + return + + try: + put_entry = settings_update.get("put-entry") + del_entry = settings_update.get("del-entry") + + if put_entry and isinstance(put_entry, dict): + bucket_key = put_entry.get("bucket-key", "") + entry_key = put_entry.get("entry-key", "") + value = put_entry.get("value") + if bucket_key and entry_key: + if value is not None: + try: + parsed = json.loads(value) if isinstance(value, str) else value + except json.JSONDecodeError: + logger.warning(f"[Urbit] Settings put-entry invalid JSON for {bucket_key}/{entry_key}") + parsed = value + logger.info(f"[Urbit] Settings hot-reload: put-entry {bucket_key}/{entry_key}") + if hasattr(self, "_on_settings_update") and self._on_settings_update: + await self._on_settings_update( + { + "action": "put", + "bucket_key": bucket_key, + "entry_key": entry_key, + "value": parsed, + } + ) + else: + logger.debug(f"[Urbit] Settings put-entry null value for {bucket_key}/{entry_key}") + + elif del_entry and isinstance(del_entry, dict): + bucket_key = del_entry.get("bucket-key", "") + entry_key = del_entry.get("entry-key", "") + if bucket_key and entry_key: + logger.info(f"[Urbit] Settings hot-reload: del-entry {bucket_key}/{entry_key}") + if hasattr(self, "_on_settings_update") and self._on_settings_update: + await self._on_settings_update( + { + "action": "del", + "bucket_key": bucket_key, + "entry_key": entry_key, + } + ) + else: + logger.debug(f"[Urbit] Settings Store event: {json.dumps(data)[:200]}") + except Exception as e: + logger.warning(f"[Urbit] Settings event processing error: {e}") + + def set_settings_update_handler(self, handler: Callable[[dict[str, Any]], Any]) -> None: + self._on_settings_update: Callable[[dict[str, Any]], Any] | None = handler + + @property + def dedup(self) -> SSEDeduplicator: + return self._dedup + + @property + def active_subscriptions(self) -> list[str]: + return [label for label, t in self._subscriptions.items() if t and not t.done()] + + +async def run_sse_listener( + client: UrbitClient, + ship_url: str, + on_message: Callable[[ChannelMessage], Any], + status_check: Callable[[], bool], + reconnect_delay: float = 5.0, + dedup: SSEDeduplicator | None = None, +) -> None: + manager = UrbitSSEManager( + client=client, + ship_url=ship_url, + on_message=on_message, + status_check=status_check, + reconnect_delay=reconnect_delay, + dedup=dedup, + ) + await manager.start({"chat": "/~/channel/chat-store"}) + for task in manager._subscriptions.values(): + if task: + await task + + +def _is_self_message(msg: ChannelMessage, bot_ship: str) -> bool: + if not bot_ship: + return False + sender = msg.metadata.get("urbit_ship", "").lower() + return sender == bot_ship.lower() + + +def _extract_cites_from_message(msg: ChannelMessage) -> list[dict[str, str]]: + additions = msg.metadata.get("raw_additions", {}) + cites_list: list[dict[str, str]] = [] + for node_data in additions.values(): + post = node_data.get("post", {}) + contents = post.get("contents", []) + for item in contents: + if isinstance(item, dict) and "cite" in item: + cite_result = parse_cite(item) + if cite_result: + cites_list.append(cite_result) + return cites_list + + +def _detect_bot_mentions(msg: ChannelMessage, bot_ship: str) -> MentionsInfo | None: + content = msg.content or "" + + if _ALL_PATTERN.search(content): + return MentionsInfo( + mentioned_user_ids=["@all"], + is_bot_mentioned=True, + ) + + if not bot_ship: + return msg.mentions + + if not content: + return None + + mentioned = False + bot_with_tilde = f"~{bot_ship}" + + if re.search(rf"\b@{bot_ship}\b", content, re.IGNORECASE): + mentioned = True + if re.search(rf"\b{bot_with_tilde}\b", content, re.IGNORECASE): + mentioned = True + if not mentioned and "additions" in str(msg.metadata): + additions = msg.metadata.get("raw_additions", {}) + mentions_list = extract_graph_mentions(additions) + if bot_ship.lower() in (m.lower() for m in mentions_list): + mentioned = True + + if mentioned: + return MentionsInfo( + mentioned_user_ids=[f"urbit:{bot_ship}"], + is_bot_mentioned=True, + ) + + return msg.mentions + + +def _graph_update_to_channel_message( + raw: dict[str, Any], + ship_name: str, +) -> ChannelMessage: + parsed = parse_graph_update(raw) + + sender_ship = parsed.get("ship", ship_name) + resource_path = parsed.get("resource_path", "") + resource_type = parsed.get("resource_type", "chat") + index = parsed.get("index", "") + + chat_type_str = "direct" if resource_type == "dm" else "group" + chat_type = ChatType.DIRECT if resource_type == "dm" else ChatType.GROUP + + if resource_type in ("diary", "heap"): + chat_type_str = "group" + chat_type = ChatType.GROUP + + identity = ChannelIdentity( + channel_id="urbit", + channel_type=ChannelType.URBIT, + channel_user_id=ensure_ship_tilde(sender_ship), + channel_chat_id=resource_path or "unknown", + channel_message_id=str(index) if index else None, + ) + + content = parsed.get("content", "") + + timestamp = None + raw_time = parsed.get("time") + if raw_time: + try: + timestamp = datetime.fromtimestamp(float(raw_time), tz=UTC) + except (TypeError, ValueError): + timestamp = None + + metadata: dict[str, Any] = { + "urbit_resource_type": resource_type, + "urbit_ship": sender_ship, + "chat_type": chat_type_str, + } + + if "reaction" in parsed: + metadata["reaction"] = parsed["reaction"] + + additions = raw.get("graph-update", {}).get("additions", {}) + if additions: + metadata["raw_additions"] = additions + + mentions = extract_graph_mentions(additions) + mentions_info = None + if mentions: + mentions_info = MentionsInfo( + mentioned_user_ids=[f"urbit:{m}" for m in mentions], + is_bot_mentioned=False, + ) + + attachments = extract_graph_attachments(additions) + + return ChannelMessage( + identity=identity, + chat_type=chat_type, + content=content.strip(), + attachments=attachments, + mentions=mentions_info, + metadata=metadata, + timestamp=timestamp, + ) diff --git a/backend/package/yuxi/channels/adapters/urbit/poke_api.py b/backend/package/yuxi/channels/adapters/urbit/poke_api.py new file mode 100644 index 00000000..62551845 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/poke_api.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +from contextlib import asynccontextmanager +from typing import Any, TYPE_CHECKING +from collections.abc import AsyncIterator + +from yuxi.utils.logging_config import logger + +if TYPE_CHECKING: + from .client import UrbitClient + + +class HttpPokeApiClient: + def __init__(self, client: UrbitClient, channel_id: str): + self._client = client + self._channel_id = channel_id + self._closed = False + + @property + def channel_id(self) -> str: + return self._channel_id + + async def poke(self, app: str, mark: str, json_data: dict[str, Any], *, timeout: float = 15.0) -> dict[str, Any]: + if self._closed: + raise RuntimeError("HttpPokeApiClient is closed") + + payload = { + "action": "poke", + "ship": self._client.ship_name, + "app": app, + "mark": mark, + "json": json_data, + } + + r = await self._client.put( + f"/~/channel/{self._channel_id}", + json=payload, + timeout=timeout, + ) + r.raise_for_status() + return r.json() + + async def scry(self, app: str, path: str) -> dict[str, Any]: + if self._closed: + raise RuntimeError("HttpPokeApiClient is closed") + + r = await self._client.get( + f"/~/scry/{app}{path}", + timeout=10.0, + ) + r.raise_for_status() + return r.json() + + async def subscribe(self, app: str, path: str) -> None: + if self._closed: + raise RuntimeError("HttpPokeApiClient is closed") + + payload = { + "action": "subscribe", + "ship": self._client.ship_name, + "app": app, + "path": path, + } + + r = await self._client.put( + f"/~/channel/{self._channel_id}", + json=payload, + timeout=10.0, + ) + r.raise_for_status() + logger.debug(f"[Urbit] HttpPokeApi subscribed: {app}{path} on {self._channel_id}") + + async def unsubscribe(self) -> None: + if self._closed: + return + payload = { + "action": "unsubscribe", + "ship": self._client.ship_name, + } + try: + r = await self._client.put( + f"/~/channel/{self._channel_id}", + json=payload, + timeout=5.0, + ) + r.raise_for_status() + except Exception as e: + logger.debug(f"[Urbit] HttpPokeApi unsubscribe error on {self._channel_id}: {e}") + + async def delete(self) -> None: + if self._closed: + return + try: + r = await self._client.delete(f"/~/channel/{self._channel_id}") + if r.status_code not in (200, 204): + logger.debug(f"[Urbit] HttpPokeApi DELETE {self._channel_id} → {r.status_code}") + except Exception as e: + logger.debug(f"[Urbit] HttpPokeApi DELETE error on {self._channel_id}: {e}") + + async def close(self) -> None: + if self._closed: + return + await self.unsubscribe() + await self.delete() + self._closed = True + logger.debug(f"[Urbit] HttpPokeApiClient closed: {self._channel_id}") + + @property + def is_closed(self) -> bool: + return self._closed + + +@asynccontextmanager +async def with_http_poke_api( + client: UrbitClient, + channel_id: str | None = None, +) -> AsyncIterator[HttpPokeApiClient]: + import time + + cid = channel_id or f"poke-api-{int(time.time() * 1000)}" + api = HttpPokeApiClient(client, cid) + logger.debug(f"[Urbit] HttpPokeApiClient opened: {cid}") + try: + yield api + finally: + try: + await api.close() + except Exception as e: + logger.warning(f"[Urbit] HttpPokeApiClient cleanup error for {cid}: {e}") + + +async def create_http_poke_api(client: UrbitClient, channel_id: str | None = None) -> HttpPokeApiClient: + import time + + cid = channel_id or f"poke-api-{int(time.time() * 1000)}" + api = HttpPokeApiClient(client, cid) + logger.debug(f"[Urbit] HttpPokeApiClient created: {cid}") + return api diff --git a/backend/package/yuxi/channels/adapters/urbit/probe.py b/backend/package/yuxi/channels/adapters/urbit/probe.py new file mode 100644 index 00000000..abbc0e9b --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/probe.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import ipaddress +from typing import Any +from urllib.parse import urlparse + +import httpx + +from yuxi.channels.exceptions import ChannelConnectionError +from yuxi.utils.logging_config import logger + +from .client import UrbitClient + +_PRIVATE_NETS = [ + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), + ipaddress.ip_network("127.0.0.0/8"), + ipaddress.ip_network("169.254.0.0/16"), + ipaddress.ip_network("::1/128"), + ipaddress.ip_network("fc00::/7"), + ipaddress.ip_network("fe80::/10"), +] + + +def validate_urbit_url(url: str, allow_private: bool = False) -> tuple[bool, list[str]]: + warnings: list[str] = [] + + if not url: + warnings.append("URL is empty") + return False, warnings + + if not url.startswith(("http://", "https://")): + warnings.append(f"URL must start with http:// or https://, got: {url}") + return False, warnings + + try: + parsed = urlparse(url) + except Exception: + warnings.append(f"Invalid URL format: {url}") + return False, warnings + + hostname = parsed.hostname + if not hostname: + warnings.append(f"URL has no hostname: {url}") + return False, warnings + + if "@" in parsed.netloc.split("@")[0] if "@" in parsed.netloc else False: + warnings.append("URL contains credentials in the host portion") + + try: + hostname_stripped = hostname.strip("[]") + addr = ipaddress.ip_address(hostname_stripped) + for net in _PRIVATE_NETS: + if addr in net: + if allow_private: + warnings.append(f"Host {hostname} is a private network address (allowed by config)") + else: + warnings.append( + f"Host {hostname} is a private network address. Set dangerously_allow_private_network=true to bypass" + ) + return False, warnings + break + except ValueError: + pass + + return True, warnings + + +async def probe_ship(client: UrbitClient) -> dict[str, object]: + try: + r = await client.get("/~/meta") + r.raise_for_status() + logger.debug(f"[Urbit] Ship ~{client.ship_name} reachable") + return {"reachable": True, "url": client.ship_url} + except httpx.RequestError as e: + raise ChannelConnectionError(f"Ship ~{client.ship_name} unreachable") from e + + +def ssrf_policy_from_dangerously_allow_private_network( + dangerously_allow_private_network: bool = False, +) -> dict[str, Any]: + + return { + "dangerouslyAllowPrivateNetwork": dangerously_allow_private_network, + "privateNets": [str(net) for net in _PRIVATE_NETS], + "allowPrivate": dangerously_allow_private_network, + } + + +def fetch_with_ssrf_guard( + url: str, + dangerously_allow_private_network: bool = False, +) -> tuple[bool, list[str]]: + return validate_urbit_url(url, allow_private=dangerously_allow_private_network) diff --git a/backend/package/yuxi/channels/adapters/urbit/rate_limiter.py b/backend/package/yuxi/channels/adapters/urbit/rate_limiter.py new file mode 100644 index 00000000..d1ce5b76 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/rate_limiter.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import asyncio +import time + +from yuxi.utils.logging_config import logger + + +class RateLimiter: + def __init__(self, limit: int = 30, window: float = 60.0): + self._limit = limit + self._window = window + self._tokens = float(limit) + self._last_refill = time.monotonic() + self._lock = asyncio.Lock() + + async def acquire(self) -> bool: + async with self._lock: + now = time.monotonic() + elapsed = now - self._last_refill + refill = elapsed * (self._limit / self._window) + self._tokens = min(float(self._limit), self._tokens + refill) + self._last_refill = now + + if self._tokens >= 1.0: + self._tokens -= 1.0 + return True + + wait_s = (1.0 - self._tokens) * (self._window / self._limit) + logger.debug(f"[Urbit] Rate limit: token exhausted, need ~{wait_s:.1f}s") + return False + + async def wait_and_acquire(self, timeout: float = 30.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if await self.acquire(): + return True + wait_s = (1.0 - max(0, self._tokens)) * (self._window / self._limit) + await asyncio.sleep(min(wait_s, 1.0)) + logger.warning("[Urbit] Rate limiter wait timed out") + return False + + @property + def available_tokens(self) -> float: + return max(0.0, self._tokens) diff --git a/backend/package/yuxi/channels/adapters/urbit/scry.py b/backend/package/yuxi/channels/adapters/urbit/scry.py new file mode 100644 index 00000000..b019c169 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/scry.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + +import httpx + +from yuxi.channels.exceptions import ChannelConnectionError +from yuxi.utils.logging_config import logger + +if TYPE_CHECKING: + from .client import UrbitClient + + +_SCRY_BASE = "/~/scry" + + +async def scry_get(client: UrbitClient, app: str, path: str) -> Any: + url = f"{_SCRY_BASE}/{app}{path}" + try: + r = await client.get(url, timeout=15.0) + if r.status_code == 400: + logger.warning(f"[Urbit] Scry {url} returned 400 (likely no data)") + return None + if r.status_code == 404: + logger.debug(f"[Urbit] Scry {url} not found") + return None + r.raise_for_status() + return r.json() + except httpx.HTTPStatusError as e: + logger.warning(f"[Urbit] Scry {url} failed: {e.response.status_code}") + return None + except httpx.RequestError as e: + raise ChannelConnectionError(f"Scry request failed: {e}") from e + + +async def scry_contacts(client: UrbitClient) -> dict[str, Any] | None: + return await scry_get(client, "contacts", "/all.json") + + +async def scry_settings(client: UrbitClient, desk: str = "moltbot") -> dict[str, Any] | None: + return await scry_get(client, "settings", f"/desk/{desk}/all.json") + + +async def scry_channels(client: UrbitClient) -> list[str] | None: + result = await scry_get(client, "channels", "/v2") + if isinstance(result, dict): + return result.get("channels", []) + return result + + +async def scry_groups_init(client: UrbitClient) -> dict[str, Any] | None: + return await scry_get(client, "groups-ui", "/v6/init.json") + + +async def scry_groups(client: UrbitClient) -> dict[str, Any] | None: + return await scry_get(client, "groups", "/groups.json") + + +async def scry_foreigns(client: UrbitClient) -> dict[str, Any] | None: + return await scry_get(client, "groups", "/v1/foreigns.json") + + +async def scry_blocked_ships(client: UrbitClient) -> list[str] | None: + result = await scry_get(client, "chat", "/blocked.json") + if isinstance(result, dict): + return result.get("blocked", []) + return result diff --git a/backend/package/yuxi/channels/adapters/urbit/send.py b/backend/package/yuxi/channels/adapters/urbit/send.py new file mode 100644 index 00000000..5a70a24c --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/send.py @@ -0,0 +1,411 @@ +from __future__ import annotations + +import asyncio +import random +from typing import Any, TYPE_CHECKING + +import httpx + +from yuxi.channels.exceptions import ChannelAuthenticationError +from yuxi.channels.models import ChannelResponse, DeliveryResult +from yuxi.utils.logging_config import logger + +from .auth import refresh_session_if_needed +from .format import build_channel_path, build_chat_action_payload, build_poke_payload + +if TYPE_CHECKING: + from .client import UrbitClient + + +async def send_poke( + client: UrbitClient, + response: ChannelResponse, + retry_config: dict[str, Any] | None = None, + *, + ship_code: str = "", +) -> DeliveryResult: + cfg = retry_config or {} + max_retries = cfg.get("attempts", 3) + min_delay_ms = cfg.get("min_delay_ms", 500) + max_delay_ms = cfg.get("max_delay_ms", 30000) + jitter = cfg.get("jitter", 0.1) + + chat_type = response.metadata.get("chat_type", "group") + channel_path = build_channel_path(**{**response.metadata, "chat_type": chat_type}) + host_ship = client.ship_name + + payload = build_poke_payload( + host_ship=host_ship, + content=response.content, + channel_path=channel_path, + continuation=response.metadata.get("continuation", False), + reply_to=response.reply_to_message_id, + chat_type=chat_type, + ) + + last_error = None + for attempt in range(max_retries): + try: + r = await client.put( + f"/~/channel/{_build_channel_id(response)}", + json=payload, + timeout=15.0, + ) + + if r.status_code == 401 and attempt < max_retries - 1: + if ship_code: + await refresh_session_if_needed(client, ship_code) + payload["ship"] = client.ship_name + payload["json"]["envelope"]["author"] = f"~{client.ship_name}" + continue + + r.raise_for_status() + return DeliveryResult(success=True) + + except httpx.HTTPStatusError as e: + last_error = f"Urbit poke failed ({e.response.status_code})" + if e.response.status_code in (404, 403): + return DeliveryResult(success=False, error=last_error) + except httpx.RequestError as e: + last_error = f"Urbit Ship unreachable: {e}" + except ChannelAuthenticationError: + last_error = "Session recovery failed" + + delay = min(max_delay_ms / 1000, (min_delay_ms / 1000) * (2**attempt)) + delay += random.uniform(0, delay * jitter) + logger.warning(f"[Urbit] Send retry {attempt + 1}/{max_retries} after {delay:.1f}s") + await asyncio.sleep(delay) + + return DeliveryResult(success=False, error=last_error or "Send failed after retries") + + +async def send_reaction_poke( + client: UrbitClient, + chat_id: str, + msg_id: str, + emoji: str, + metadata: dict[str, Any] | None = None, + retry_config: dict[str, Any] | None = None, + *, + ship_code: str = "", +) -> DeliveryResult: + return await _send_chat_action( + client=client, + chat_id=chat_id, + action="add-react", + msg_id=msg_id, + metadata=metadata, + retry_config=retry_config, + emoji=emoji, + ship_code=ship_code, + ) + + +async def send_del_reaction_poke( + client: UrbitClient, + chat_id: str, + msg_id: str, + emoji: str, + metadata: dict[str, Any] | None = None, + retry_config: dict[str, Any] | None = None, + *, + ship_code: str = "", +) -> DeliveryResult: + return await _send_chat_action( + client=client, + chat_id=chat_id, + action="del-react", + msg_id=msg_id, + metadata=metadata, + retry_config=retry_config, + emoji=emoji, + ship_code=ship_code, + ) + + +async def send_edit_poke( + client: UrbitClient, + chat_id: str, + msg_id: str, + content: str, + metadata: dict[str, Any] | None = None, + retry_config: dict[str, Any] | None = None, + *, + ship_code: str = "", +) -> DeliveryResult: + return await _send_chat_action( + client=client, + chat_id=chat_id, + action="edit", + msg_id=msg_id, + metadata=metadata, + retry_config=retry_config, + content=content, + ship_code=ship_code, + ) + + +async def send_delete_poke( + client: UrbitClient, + chat_id: str, + msg_id: str, + metadata: dict[str, Any] | None = None, + retry_config: dict[str, Any] | None = None, + *, + ship_code: str = "", +) -> DeliveryResult: + return await _send_chat_action( + client=client, + chat_id=chat_id, + action="del", + msg_id=msg_id, + metadata=metadata, + retry_config=retry_config, + ship_code=ship_code, + ) + + +async def send_block_ship_poke( + client: UrbitClient, + ship: str, + *, + ship_code: str = "", +) -> DeliveryResult: + target = ship.lstrip("~") + payload = { + "action": "poke", + "ship": client.ship_name, + "app": "chat", + "mark": "chat-block-ship", + "json": {"ship": f"~{target}"}, + } + + try: + r = await client.put( + "/~/channel/chat-0", + json=payload, + timeout=10.0, + ) + if r.status_code == 401 and ship_code: + from .auth import refresh_session_if_needed + + await refresh_session_if_needed(client, ship_code) + payload["ship"] = client.ship_name + r = await client.put( + "/~/channel/chat-0", + json=payload, + timeout=10.0, + ) + r.raise_for_status() + logger.info(f"[Urbit] Ship blocked via poke: ~{target}") + return DeliveryResult(success=True) + except httpx.HTTPStatusError as e: + return DeliveryResult(success=False, error=f"block-ship failed ({e.response.status_code})") + except httpx.RequestError as e: + return DeliveryResult(success=False, error=f"Urbit Ship unreachable: {e}") + except Exception as e: + return DeliveryResult(success=False, error=str(e)) + + +async def send_unblock_ship_poke( + client: UrbitClient, + ship: str, + *, + ship_code: str = "", +) -> DeliveryResult: + target = ship.lstrip("~") + payload = { + "action": "poke", + "ship": client.ship_name, + "app": "chat", + "mark": "chat-unblock-ship", + "json": {"ship": f"~{target}"}, + } + + try: + r = await client.put( + "/~/channel/chat-0", + json=payload, + timeout=10.0, + ) + if r.status_code == 401 and ship_code: + from .auth import refresh_session_if_needed + + await refresh_session_if_needed(client, ship_code) + payload["ship"] = client.ship_name + r = await client.put( + "/~/channel/chat-0", + json=payload, + timeout=10.0, + ) + r.raise_for_status() + logger.info(f"[Urbit] Ship unblocked via poke: ~{target}") + return DeliveryResult(success=True) + except httpx.HTTPStatusError as e: + return DeliveryResult(success=False, error=f"unblock-ship failed ({e.response.status_code})") + except httpx.RequestError as e: + return DeliveryResult(success=False, error=f"Urbit Ship unreachable: {e}") + except Exception as e: + return DeliveryResult(success=False, error=str(e)) + + +async def _send_chat_action( + *, + client: UrbitClient, + chat_id: str, + action: str, + msg_id: str, + metadata: dict[str, Any] | None = None, + retry_config: dict[str, Any] | None = None, + emoji: str | None = None, + content: str | None = None, + ship_code: str = "", +) -> DeliveryResult: + cfg = retry_config or {} + max_retries = cfg.get("attempts", 2) + min_delay_ms = cfg.get("min_delay_ms", 300) + max_delay_ms = cfg.get("max_delay_ms", 10000) + jitter = cfg.get("jitter", 0.1) + + meta = metadata or {} + chat_type = meta.get("chat_type", "group") + channel_path = build_channel_path(**{**meta, "chat_type": chat_type}) + host_ship = client.ship_name + + payload = build_chat_action_payload( + host_ship=host_ship, + channel_path=channel_path, + action=action, + msg_id=msg_id, + emoji=emoji, + content=content, + ) + + last_error = None + for attempt in range(max_retries): + try: + r = await client.put( + f"/~/channel/{chat_id}", + json=payload, + timeout=10.0, + ) + + if r.status_code == 401 and attempt < max_retries - 1: + if ship_code: + await refresh_session_if_needed(client, ship_code) + payload["ship"] = client.ship_name + continue + + r.raise_for_status() + return DeliveryResult(success=True) + + except httpx.HTTPStatusError as e: + last_error = f"Urbit chat action '{action}' failed ({e.response.status_code})" + if e.response.status_code in (404, 403): + return DeliveryResult(success=False, error=last_error) + except httpx.RequestError as e: + last_error = f"Urbit Ship unreachable: {e}" + + delay = min(max_delay_ms / 1000, (min_delay_ms / 1000) * (2**attempt)) + delay += random.uniform(0, delay * jitter) + await asyncio.sleep(delay) + + return DeliveryResult(success=False, error=last_error or f"Chat action '{action}' failed after retries") + + +async def _send_media_poke( + client: UrbitClient, + response: ChannelResponse, + payload: dict[str, Any], + retry_config: dict[str, Any] | None = None, +) -> DeliveryResult: + cfg = retry_config or {} + max_retries = cfg.get("attempts", 2) + min_delay_ms = cfg.get("min_delay_ms", 500) + max_delay_ms = cfg.get("max_delay_ms", 30000) + jitter = cfg.get("jitter", 0.1) + + last_error = None + for attempt in range(max_retries): + try: + r = await client.put( + f"/~/channel/{_build_channel_id(response)}", + json=payload, + timeout=15.0, + ) + r.raise_for_status() + return DeliveryResult(success=True) + except httpx.HTTPStatusError as e: + last_error = f"Urbit media send failed ({e.response.status_code})" + if e.response.status_code in (404, 403): + return DeliveryResult(success=False, error=last_error) + except httpx.RequestError as e: + last_error = f"Urbit Ship unreachable: {e}" + + delay = min(max_delay_ms / 1000, (min_delay_ms / 1000) * (2**attempt)) + delay += random.uniform(0, delay * jitter) + await asyncio.sleep(delay) + + return DeliveryResult(success=False, error=last_error or "Media send failed after retries") + + +async def send_chat_action( + client: UrbitClient, + chat_id: str, + metadata: dict[str, Any], + action: str = "typing", + *, + ship_code: str = "", +) -> DeliveryResult: + meta = metadata or {} + chat_type = meta.get("chat_type", "group") + channel_path = build_channel_path(**{**meta, "chat_type": chat_type}) + host_ship = client.ship_name + + payload = build_chat_action_payload( + host_ship=host_ship, + channel_path=channel_path, + action=action, + msg_id="", + ) + + try: + r = await client.put( + f"/~/channel/{chat_id}", + json=payload, + timeout=10.0, + ) + + if r.status_code == 401 and ship_code: + await refresh_session_if_needed(client, ship_code) + payload["ship"] = client.ship_name + r = await client.put( + f"/~/channel/{chat_id}", + json=payload, + timeout=10.0, + ) + + r.raise_for_status() + return DeliveryResult(success=True) + + except httpx.HTTPStatusError as e: + return DeliveryResult(success=False, error=f"Chat action '{action}' failed ({e.response.status_code})") + except httpx.RequestError as e: + return DeliveryResult(success=False, error=f"Urbit Ship unreachable: {e}") + except Exception as e: + return DeliveryResult(success=False, error=str(e)) + + +async def _build_channel_id(response: ChannelResponse) -> str: + meta = response.metadata + chat_type = meta.get("chat_type", "group") + + if chat_type == "direct": + target = meta.get("target_ship", "").lstrip("~") + host = meta.get("host_ship", "").lstrip("~") + return f"~{target}/dm--{host}" + + host = meta.get("host_ship", "").lstrip("~") + group = meta.get("group_name", "unknown") + ch_type = meta.get("channel_type", "chat") + return f"~{host}/{group}/{ch_type}" diff --git a/backend/package/yuxi/channels/adapters/urbit/session.py b/backend/package/yuxi/channels/adapters/urbit/session.py new file mode 100644 index 00000000..ac14b0ec --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/session.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from collections import defaultdict + +from yuxi.channels.models import ChannelMessage +from yuxi.utils.logging_config import logger + + +def resolve_thread_key(msg: ChannelMessage, default_agent_id: str = "ChatbotAgent") -> str: + chat_type = msg.metadata.get("chat_type", "group") + chat_id = msg.identity.channel_chat_id + + if chat_type == "direct": + return f"agent:{default_agent_id}:urbit:direct:{chat_id}" + + group_name = msg.metadata.get("group_name", chat_id) + ch_type = msg.metadata.get("urbit_resource_type", "chat") + return f"agent:{default_agent_id}:urbit:group:{group_name}:{ch_type}" + + +def resolve_session_route(msg: ChannelMessage, default_agent_id: str = "ChatbotAgent") -> str: + return resolve_thread_key(msg, default_agent_id) + + +_unsafe_sessions: defaultdict[str, set[str]] = defaultdict(set) + + +def detect_unsafe_session(msg: ChannelMessage) -> list[str]: + chat_type = msg.metadata.get("chat_type", "group") + if chat_type != "direct": + return [] + + chat_id = msg.identity.channel_chat_id + participant_key = f"dm:{chat_id}" + sender_ship = msg.metadata.get("urbit_ship", "") + + if not sender_ship: + return [] + + _unsafe_sessions[participant_key].add(sender_ship) + participants = _unsafe_sessions[participant_key] + + if len(participants) > 2: + logger.warning( + f"[Urbit] Unsafe DM session detected: " + f"{participant_key} has {len(participants)} " + f"participants: {participants}" + ) + return list(participants) + + return [] + + +def get_unsafe_sessions() -> dict[str, set[str]]: + return dict(_unsafe_sessions) + + +def clear_unsafe_sessions() -> None: + _unsafe_sessions.clear() diff --git a/backend/package/yuxi/channels/adapters/urbit/settings.py b/backend/package/yuxi/channels/adapters/urbit/settings.py new file mode 100644 index 00000000..c55cd7da --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/settings.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + +from yuxi.utils.logging_config import logger + +if TYPE_CHECKING: + from .client import UrbitClient + + +_MIGRATABLE_KEYS = [ + "dmAllowlist", + "groupChannels", + "autoDiscoverChannels", + "ownerShip", + "autoAcceptGroupInvites", + "autoAcceptDmInvites", + "groupInviteAllowlist", +] + + +class SettingsStore: + def __init__(self, client: UrbitClient, desk: str = "moltbot"): + self._client = client + self._desk = desk + self._store: dict[str, Any] = {} + self._file_config: dict[str, Any] = {} + self._loaded = False + self._version: int = 0 + self._key_versions: dict[str, int] = {} + + async def load(self, file_config: dict[str, Any] | None = None) -> dict[str, Any]: + if file_config: + self._file_config = file_config + + try: + from .scry import scry_settings + + store_data = await scry_settings(self._client, self._desk) + if store_data and isinstance(store_data, dict): + self._store = store_data + self._version += 1 + logger.info(f"[Urbit] Settings Store loaded from desk '{self._desk}' (v{self._version})") + else: + logger.debug(f"[Urbit] Settings Store empty or not found for desk '{self._desk}'") + except Exception as e: + logger.warning(f"[Urbit] Failed to load Settings Store: {e}") + + self._loaded = True + return self._store + + def get(self, key: str, default: Any = None) -> Any: + store_val = self._store.get(key) + if store_val is not None: + if isinstance(store_val, list) and len(store_val) == 0: + return [] + return store_val + + return self._file_config.get(key, default) + + def get_all(self) -> dict[str, Any]: + merged = dict(self._file_config) + for key, value in self._store.items(): + if value is not None: + merged[key] = value + return merged + + def merge_key(self, key: str, value: Any, expected_version: int | None = None) -> bool: + if expected_version is not None: + current_version = self._key_versions.get(key, 0) + if expected_version < current_version: + logger.debug( + f"[Urbit] Settings merge rejected for '{key}': version {expected_version} < {current_version}" + ) + return False + + store = self._store.copy() + store[key] = value + self._store = store + self._key_versions[key] = self._key_versions.get(key, 0) + 1 + return True + + async def reload(self) -> None: + self._version += 1 + await self.load(self._file_config) + + async def build_settings_migrations(self) -> list[dict[str, Any]]: + migrations: list[dict[str, Any]] = [] + for key in _MIGRATABLE_KEYS: + if self.should_migrate_setting(key): + migrations.append( + { + "bucket_key": "moltbot", + "entry_key": key, + "value": self._file_config.get(key), + } + ) + return migrations + + def should_migrate_setting(self, key: str) -> bool: + file_val = self._file_config.get(key) + if file_val is None: + return False + store_val = self._store.get(key) + return store_val is None + + async def poke_migration(self, bucket_key: str, entry_key: str, value: Any) -> bool: + import json as _json + + try: + value_str = _json.dumps(value) if not isinstance(value, str) else value + payload = { + "action": "poke", + "ship": self._client.ship_name, + "app": "settings", + "mark": "settings-event", + "json": { + "put-entry": { + "bucket-key": bucket_key, + "entry-key": entry_key, + "value": value_str, + } + }, + } + r = await self._client.put("/~/channel/settings-store", json=payload, timeout=10.0) + r.raise_for_status() + logger.info(f"[Urbit] Migrated setting: {entry_key} → Settings Store") + return True + except Exception as e: + logger.warning(f"[Urbit] Migration failed for {entry_key}: {e}") + return False + + @property + def is_loaded(self) -> bool: + return self._loaded + + @property + def version(self) -> int: + return self._version + + +def parse_channel_rules(raw: Any) -> dict[str, Any] | None: + if raw is None: + return None + if isinstance(raw, dict): + return raw + if isinstance(raw, str): + import json as _json + + try: + return _json.loads(raw) + except _json.JSONDecodeError: + logger.warning("[Urbit] parse_channel_rules: invalid JSON string") + return None + return None + + +def parse_pending_approvals(raw: Any) -> list[dict[str, Any]] | None: + if raw is None: + return None + if isinstance(raw, list): + return raw + if isinstance(raw, str): + import json as _json + + try: + return _json.loads(raw) + except _json.JSONDecodeError: + logger.warning("[Urbit] parse_pending_approvals: invalid JSON string") + return None + return None + + +def serialize_to_json_value(value: Any) -> str: + import json as _json + + return _json.dumps(value) diff --git a/backend/package/yuxi/channels/adapters/urbit/setup.py b/backend/package/yuxi/channels/adapters/urbit/setup.py new file mode 100644 index 00000000..c15c8399 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/setup.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +from typing import Any + +from yuxi.utils.logging_config import logger + + +def run_setup_wizard(config: dict[str, Any], interactive: bool = True) -> dict[str, Any]: + result: dict[str, Any] = {} + + ship_url = config.get("ship_url", "") + if not ship_url and interactive: + ship_url = input("Enter your Urbit ship URL (e.g. http://localhost:8080): ").strip() + + from .probe import validate_urbit_url + + valid, warnings = validate_urbit_url(ship_url) + if warnings: + for w in warnings: + logger.warning(f"[Urbit Setup] {w}") + + if not valid and interactive: + allow_private = input("URL validation failed. Allow private network connections? (y/n): ").strip().lower() + if allow_private == "y": + config["network"] = config.get("network", {}) + config["network"]["dangerously_allow_private_network"] = True + result["dangerously_allow_private_network"] = True + + result["ship_url"] = ship_url + + ship_name = config.get("ship_name", "") + if not ship_name and interactive: + ship_name = input("Enter your Urbit ship name (without ~): ").strip() + result["ship_name"] = ship_name + + ship_code = config.get("ship_code", "") + if not ship_code and interactive: + ship_code = input("Enter your ship +code: ").strip() + result["ship_code"] = ship_code + + group_channels = config.get("group_channels", []) + if not group_channels and interactive: + channels_input = input( + "Group channels to monitor (comma-separated, e.g. chat/some-group, diary/some-diary): " + ).strip() + if channels_input: + group_channels = [c.strip() for c in channels_input.split(",") if c.strip()] + result["group_channels"] = group_channels + + dm_allowlist = config.get("dm_allowlist", []) + if not dm_allowlist and interactive: + allowlist_input = input( + "DM whitelist ships (comma-separated with urbit: prefix, e.g. urbit:zod, urbit:bus): " + ).strip() + if allowlist_input: + dm_allowlist = [s.strip() for s in allowlist_input.split(",") if s.strip()] + result["dm_allowlist"] = dm_allowlist + + auto_discover = config.get("auto_discover_channels", True) + result["auto_discover_channels"] = auto_discover + + logger.info("[Urbit Setup] Configuration complete") + return result + + +def quick_setup(ship_url: str, ship_name: str, ship_code: str, **kw) -> dict[str, Any]: + config: dict[str, Any] = { + "ship_url": ship_url, + "ship_name": ship_name, + "ship_code": ship_code, + "group_channels": kw.pop("group_channels", []), + "dm_allowlist": kw.pop("dm_allowlist", []), + "auto_discover_channels": kw.pop("auto_discover", True), + } + config.update(kw) + return run_setup_wizard(config, interactive=False) + + +def get_setup_schema() -> dict[str, Any]: + return { + "name": "urbit", + "display_name": "Urbit (Tlon)", + "description": "Connect to your Urbit ship for group chat and DM", + "required_fields": [ + {"key": "ship_url", "label": "Ship URL", "type": "url", "placeholder": "http://localhost:8080"}, + {"key": "ship_name", "label": "Ship Name", "type": "text", "placeholder": "zod"}, + {"key": "ship_code", "label": "Ship +code", "type": "password", "placeholder": "XXXX-XXXX-XXXX-XXXX"}, + ], + "optional_fields": [ + {"key": "dm_allowlist", "label": "DM Allowlist", "type": "list"}, + {"key": "group_channels", "label": "Group Channels", "type": "list"}, + { + "key": "autoDiscoverChannels", + "label": "Auto-discover Channels", + "type": "boolean", + "default": True, + }, + { + "key": "dm_policy", + "label": "DM Policy", + "type": "select", + "options": ["open", "disabled", "pairing", "allowlist"], + "default": "pairing", + }, + { + "key": "group_policy", + "label": "Group Policy", + "type": "select", + "options": ["open", "disabled", "allowlist"], + "default": "open", + "description": "Controls how the bot handles incoming group messages", + }, + { + "key": "owner_ship", + "label": "Owner Ship", + "type": "text", + "description": "Ship that receives admin notifications", + }, + { + "key": "autoAcceptDmInvites", + "label": "Auto Accept DM Invites", + "type": "boolean", + "default": False, + }, + { + "key": "autoAcceptGroupInvites", + "label": "Auto Accept Group Invites", + "type": "boolean", + "default": False, + }, + { + "key": "groupInviteAllowlist", + "label": "Group Invite Allowlist", + "type": "list", + "description": "Group paths allowed for auto-accept", + }, + { + "key": "streaming", + "label": "Streaming Mode", + "type": "select", + "options": ["off", "partial"], + "default": "off", + "description": "Controls how streaming responses are delivered", + }, + { + "key": "configWrites", + "label": "Config Writes", + "type": "object", + "description": "Custom config values pushed to Settings Store on connect", + }, + { + "key": "authorization.channel_rules", + "label": "Channel Rules", + "type": "object", + "description": 'Per-channel authorization rules, e.g. {"chat/my-group": {"allow": ["zod"]}}', + }, + { + "key": "authorization.default_authorized_ships", + "label": "Default Authorized Ships", + "type": "list", + "description": "Ships authorized across all channels by default", + }, + { + "key": "network.dangerously_allow_private_network", + "label": "Allow Private Network", + "type": "boolean", + "default": False, + "description": "Allow connections to private/local network addresses", + }, + { + "key": "trustedLocalFileRoots", + "label": "Trusted Local File Roots", + "type": "list", + "description": "Trusted directory paths for file operations", + }, + { + "key": "timeoutSeconds", + "label": "Request Timeout (s)", + "type": "number", + "default": 30, + "min": 5, + "max": 300, + }, + { + "key": "showModelSignature", + "label": "Show Model Signature", + "type": "boolean", + "default": False, + }, + { + "key": "responsePrefix", + "label": "Response Prefix", + "type": "text", + "description": "Optional prefix added before bot responses", + }, + { + "key": "commands.native", + "label": "Enable Native Commands", + "type": "boolean", + "default": False, + }, + { + "key": "commands.nativeSkills", + "label": "Native Command Skills", + "type": "list", + }, + ], + } diff --git a/backend/package/yuxi/channels/adapters/urbit/story.py b/backend/package/yuxi/channels/adapters/urbit/story.py new file mode 100644 index 00000000..1e512399 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/story.py @@ -0,0 +1,316 @@ +from __future__ import annotations + +import re +from typing import Any + +_HEADER_RE = re.compile(r"^(#{1,4})\s+(.+)$", re.MULTILINE) +_BOLD_RE = re.compile(r"\*\*(.+?)\*\*") +_ITALIC_RE = re.compile(r"\*(.+?)\*") +_STRIKE_RE = re.compile(r"~~(.+?)~~") +_INLINE_CODE_RE = re.compile(r"`([^`]+)`") +_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)") +_CODE_BLOCK_RE = re.compile(r"```(\w*)\n?(.*?)```", re.DOTALL) +_BLOCKQUOTE_RE = re.compile(r"^>\s?(.+)$", re.MULTILINE) +_UNORDERED_LIST_RE = re.compile(r"^[-*+]\s+(.+)$", re.MULTILINE) +_ORDERED_LIST_RE = re.compile(r"^\d+\.\s+(.+)$", re.MULTILINE) +_SHIP_RE = re.compile(r"(? list[dict[str, Any]]: + if not content: + return [{"text": ""}] + + if ( + "```" in content + or "**" in content + or "#" in content + or ">" in content + or "[" in content + or "~" in content + or _RULE_RE.search(content) + ): + return _parse_markdown_story(content) + + lines = content.split("\n") + if len(lines) == 1: + return [{"text": content}] + + story: list[dict[str, Any]] = [] + for line in lines: + if not line.strip(): + story.append({"break": None}) + else: + story.append({"text": line}) + return story + + +def _parse_markdown_story(content: str) -> list[dict[str, Any]]: + story: list[dict[str, Any]] = [] + pos = 0 + + while pos < len(content): + if content[pos:].startswith("```"): + end = content.find("```", pos + 3) + if end == -1: + story.append({"text": content[pos:]}) + break + block_text = content[pos + 3 : end] + lang = "" + if "\n" in block_text: + first_line, rest = block_text.split("\n", 1) + if first_line.strip() and not first_line.strip().startswith(" "): + lang = first_line.strip() + block_text = rest + story.append({"code": {"code": block_text.strip(), "lang": lang}}) + pos = end + 3 + while pos < len(content) and content[pos] == "\n": + pos += 1 + continue + + if content[pos:].startswith(("# ", "## ", "### ", "#### ")): + end = content.find("\n", pos) + if end == -1: + end = len(content) + line = content[pos:end] + level = 0 + for ch in line: + if ch == "#": + level += 1 + else: + break + tag = f"h{level}" if level <= 4 else "h4" + story.append({"header": {"content": line[level:].strip(), "tag": tag}}) + pos = end + 1 + continue + + if content[pos:].startswith("> "): + end = content.find("\n", pos) + if end == -1: + end = len(content) + quote_text = content[pos + 2 : end] + story.append({"blockquote": _parse_inline(quote_text)}) + pos = end + 1 + continue + + m = re.match(r"^[-*+]\s+", content[pos:]) + if m: + items: list[str] = [] + while pos < len(content): + m2 = re.match(r"^[-*+]\s+(.+)$", content[pos:], re.MULTILINE) + if not m2: + break + items.append(m2.group(1)) + end = content.find("\n", pos + m2.end()) + if end == -1: + pos = len(content) + else: + pos = end + 1 + if items: + story.append({"listing": {"type": "unordered", "items": items, "contents": []}}) + continue + + m = re.match(r"^\d+\.\s+", content[pos:]) + if m: + items: list[str] = [] + while pos < len(content): + m2 = re.match(r"^\d+\.\s+(.+)$", content[pos:], re.MULTILINE) + if not m2: + break + items.append(m2.group(1)) + end = content.find("\n", pos + m2.end()) + if end == -1: + pos = len(content) + else: + pos = end + 1 + if items: + story.append({"listing": {"type": "ordered", "items": items, "contents": []}}) + continue + + m = _RULE_RE.match(content[pos:]) + if m: + end = content.find("\n", pos) + if end == -1: + end = len(content) + story.append({"block": {"rule": None}}) + pos = end + 1 + continue + + end = content.find("\n", pos) + if end == -1: + end = len(content) + line = content[pos:end] + if line.strip(): + story.append(_parse_inline(line)) + pos = end + 1 + + return story + + +def _parse_inline(text: str) -> dict[str, Any]: + remaining = text + inlines: list[dict[str, Any]] = [] + + while remaining: + has_match = False + earliest = len(remaining) + match_type = None + match_data = None + + for m in _BOLD_RE.finditer(remaining): + if m.start() < earliest: + earliest = m.start() + match_type = "bold" + match_data = m + has_match = True + + for m in _ITALIC_RE.finditer(remaining): + if m.start() < earliest: + earliest = m.start() + match_type = "italic" + match_data = m + has_match = True + + for m in _STRIKE_RE.finditer(remaining): + if m.start() < earliest: + earliest = m.start() + match_type = "strike" + match_data = m + has_match = True + + for m in _INLINE_CODE_RE.finditer(remaining): + if m.start() < earliest: + earliest = m.start() + match_type = "code" + match_data = m + has_match = True + + for m in _LINK_RE.finditer(remaining): + if m.start() < earliest: + earliest = m.start() + match_type = "link" + match_data = m + has_match = True + + for m in _SHIP_RE.finditer(remaining): + if m.start() < earliest: + earliest = m.start() + match_type = "ship_ref" + match_data = m + has_match = True + + if not has_match: + if remaining.strip(): + inlines.append({"text": remaining}) + break + + if earliest > 0: + inlines.append({"text": remaining[:earliest]}) + + if match_type == "bold": + inlines.append({"bold": [_parse_inline(match_data.group(1))]}) + elif match_type == "italic": + inlines.append({"italics": [_parse_inline(match_data.group(1))]}) + elif match_type == "strike": + inlines.append({"strike": [_parse_inline(match_data.group(1))]}) + elif match_type == "code": + inlines.append({"inline-code": match_data.group(1)}) + elif match_type == "link": + inlines.append({"link": {"href": match_data.group(2), "content": match_data.group(1)}}) + elif match_type == "ship_ref": + inlines.append({"ship": match_data.group(1)}) + + remaining = remaining[match_data.end() :] + + if len(inlines) == 1: + return inlines[0] + + has_only_text = all("text" in item for item in inlines) + if has_only_text: + return {"text": "".join(item["text"] for item in inlines)} + + return {"text": "".join(_inline_to_text(item) for item in inlines)} + + +def _inline_to_text(item: dict[str, Any]) -> str: + if "text" in item: + return item["text"] + if "bold" in item: + inner = item["bold"] + if isinstance(inner, list): + return "".join(_inline_to_text(i) for i in inner) + return str(inner) + if "italics" in item: + inner = item["italics"] + if isinstance(inner, list): + return "".join(_inline_to_text(i) for i in inner) + return str(inner) + if "strike" in item: + inner = item["strike"] + if isinstance(inner, list): + return "".join(_inline_to_text(i) for i in inner) + return str(inner) + if "inline-code" in item: + return str(item["inline-code"]) + if "link" in item: + link = item["link"] + return link.get("content", link.get("href", "")) if isinstance(link, dict) else str(link) + if "ship" in item: + return item["ship"] + return "" + + +def story_to_plain_text(story: list[dict[str, Any]]) -> str: + parts: list[str] = [] + for item in story: + if "text" in item: + parts.append(item["text"]) + elif "bold" in item: + parts.append("".join(_inline_to_text(i) for i in item["bold"])) + elif "italics" in item: + parts.append("".join(_inline_to_text(i) for i in item["italics"])) + elif "strike" in item: + parts.append("".join(_inline_to_text(i) for i in item["strike"])) + elif "inline-code" in item: + parts.append(item["inline-code"]) + elif "blockquote" in item: + parts.append(_story_to_text_recursive(item["blockquote"])) + elif "header" in item: + parts.append(item["header"].get("content", "")) + elif "code" in item: + code_data = item["code"] + if isinstance(code_data, dict): + parts.append(code_data.get("code", "")) + elif "link" in item: + link = item["link"] + if isinstance(link, dict): + parts.append(link.get("content", link.get("href", ""))) + elif "break" in item: + parts.append("\n") + elif "ship" in item: + parts.append(item["ship"]) + elif "listing" in item: + listing = item["listing"] + if isinstance(listing, dict): + for list_item in listing.get("items", []): + parts.append(f"- {list_item}\n") + return "".join(parts) + + +def _story_to_text_recursive(items: list[dict[str, Any]]) -> str: + parts: list[str] = [] + for item in items: + if "text" in item: + parts.append(item["text"]) + elif "bold" in item: + parts.append("".join(_inline_to_text(i) for i in item["bold"])) + elif "italics" in item: + parts.append("".join(_inline_to_text(i) for i in item["italics"])) + elif "inline-code" in item: + parts.append(item["inline-code"]) + elif "link" in item: + link = item["link"] + parts.append(link.get("content", "") if isinstance(link, dict) else str(link)) + elif "break" in item: + parts.append("\n") + return "".join(parts) diff --git a/backend/package/yuxi/channels/adapters/urbit/summarizer.py b/backend/package/yuxi/channels/adapters/urbit/summarizer.py new file mode 100644 index 00000000..0a53184a --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/summarizer.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + +from yuxi.utils.logging_config import logger + +if TYPE_CHECKING: + from .history import MessageCache + + +class Summarizer: + def __init__(self, message_cache: MessageCache | None = None): + self._trigger_patterns = [ + "summarize", + "catch up", + "tldr", + "summary", + "what did i miss", + "summarise", + ] + self._message_cache = message_cache + + def is_summarization_request(self, content: str) -> bool: + lowered = content.lower().strip() + return any(pattern in lowered for pattern in self._trigger_patterns) + + def build_summary_request( + self, + channel_id: str, + history: list[dict[str, Any]], + chat_type: str = "group", + ) -> dict[str, Any]: + messages_text = "\n".join(f"[{msg.get('author', 'unknown')}]: {msg.get('content', '')}" for msg in history) + + system_prompt = ( + f"You are summarizing a {chat_type} conversation from an Urbit channel. " + f"Provide a concise summary structured as follows:\n" + f"(1) Main topics - the key subjects discussed\n" + f"(2) Key decisions - any decisions, conclusions, or agreements reached\n" + f"(3) Action items - tasks, follow-ups, or next steps mentioned\n" + f"(4) Notable participants - key contributors and their roles" + ) + + return { + "system_prompt": system_prompt, + "channel_id": channel_id, + "message_count": len(history), + "messages": messages_text, + "instruction": ( + "Please summarize these messages with the following structure:\n" + "1. Main topics\n" + "2. Key decisions\n" + "3. Action items\n" + "4. Notable participants" + ), + } + + def get_recent_history(self, chat_id: str, max_messages: int = 50) -> list[dict[str, Any]]: + if not self._message_cache: + return [] + return self._message_cache.get_recent(chat_id, max_messages) + + def auto_detect_and_respond(self, chat_id: str, content: str) -> dict[str, Any] | None: + if not self.is_summarization_request(content): + return None + + history = self.get_recent_history(chat_id) + if not history: + logger.info(f"[Urbit] Summarizer: no history available for {chat_id}") + return None + + logger.info(f"[Urbit] Summarizer: detected request in {chat_id}, {len(history)} recent messages") + return self.build_summary_request(chat_id, history) + + def bind_cache(self, message_cache: MessageCache) -> None: + self._message_cache = message_cache diff --git a/backend/package/yuxi/channels/adapters/urbit/targets.py b/backend/package/yuxi/channels/adapters/urbit/targets.py new file mode 100644 index 00000000..18bf0398 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/targets.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + pass + + +def parse_tlon_target(target: str) -> dict[str, str]: + target = target.strip().lstrip("~") + + if target.startswith("tlon:dm/"): + return {"chat_type": "direct", "peer_id": target[len("tlon:dm/") :]} + + if target.startswith("dm/"): + return {"chat_type": "direct", "peer_id": target[3:]} + + if target.startswith("group:chat/"): + parts = target[len("group:chat/") :].split("/") + if len(parts) >= 2: + return {"chat_type": "group", "group_name": "/".join(parts), "channel_type": "chat"} + return {"chat_type": "group", "group_name": target[len("group:chat/") :]} + + if target.startswith("group:"): + return {"chat_type": "group", "group_name": target[6:]} + + nest_match = target.split("/") + if len(nest_match) >= 2 and nest_match[0] in ("chat", "diary", "heap"): + return { + "chat_type": "group", + "channel_type": nest_match[0], + "group_name": "/".join(nest_match[1:]), + } + + return {"chat_type": "direct", "peer_id": target} + + +def format_target_hint(chat_type: str, channel_id: str, host_ship: str = "") -> str: + if chat_type == "direct": + return f"dm/~{channel_id.lstrip('~')}" + host = host_ship.lstrip("~") + return f"~{host}/{channel_id} | chat/~{host}/{channel_id} | group:~{host}/{channel_id}" diff --git a/backend/package/yuxi/channels/adapters/urbit/threads.py b/backend/package/yuxi/channels/adapters/urbit/threads.py new file mode 100644 index 00000000..90c1aa76 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/threads.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + +from yuxi.utils.logging_config import logger + +if TYPE_CHECKING: + from .client import UrbitClient + + +class ThreadManager: + def __init__(self, max_history: int = 20, context_lines: int = 10): + self._participated: set[str] = set() + self._max_history = max_history + self._context_lines = context_lines + + def is_thread_reply(self, raw_data: dict[str, Any]) -> bool: + graph_update = raw_data.get("graph-update", {}) + additions = graph_update.get("additions", {}) + for node_data in additions.values(): + post = node_data.get("post", {}) + contents = post.get("contents", []) + for item in contents: + if isinstance(item, dict) and "reply" in item: + return True + return False + + def get_reply_parent_id(self, raw_data: dict[str, Any]) -> str | None: + graph_update = raw_data.get("graph-update", {}) + additions = graph_update.get("additions", {}) + for node_data in additions.values(): + post = node_data.get("post", {}) + contents = post.get("contents", []) + for item in contents: + if isinstance(item, dict) and "reply" in item: + reply_data = item["reply"] + return reply_data.get("id") or reply_data + return None + + def add_participated(self, thread_id: str) -> None: + self._participated.add(thread_id) + + def has_participated(self, thread_id: str) -> bool: + return thread_id in self._participated + + async def fetch_thread_history( + self, client: UrbitClient, ship_name: str, resource_path: str, parent_id: str + ) -> list[dict[str, Any]]: + from .scry import scry_get + + try: + path = f"/graph/{ship_name}/{resource_path}" + result = await scry_get(client, "graph", path) + if not result or not isinstance(result, dict): + return [] + + nodes = result.get("graph", {}).get("nodes", {}) + thread_messages: list[dict[str, Any]] = [] + + for _node_id, node in nodes.items(): + post = node.get("post", {}) + if not post: + continue + if _is_child_of(parent_id, node): + thread_messages.append(_extract_message_content(node)) + + thread_messages.sort(key=lambda m: m.get("time", 0)) + return thread_messages[-self._max_history :] + + except Exception as e: + logger.warning(f"[Urbit] Failed to fetch thread history: {e}") + return [] + + def build_thread_context(self, history: list[dict[str, Any]]) -> str: + recent = history[-self._context_lines :] + if not recent: + return "" + lines: list[str] = [ + f"[Thread conversation - {len(recent)} previous replies...]", + "[Previous messages]", + ] + for msg in recent: + author = msg.get("author", "unknown") + content = msg.get("content", "") + lines.append(f"~{author}: {content}") + lines.append("[Current message]") + return "\n".join(lines) + + +def _is_child_of(parent_id: str, node: dict[str, Any]) -> bool: + children = node.get("children", {}) + if parent_id in children: + return True + for child_node in children.values(): + if isinstance(child_node, dict) and _is_child_of(parent_id, child_node): + return True + return False + + +def _extract_message_content(node: dict[str, Any]) -> dict[str, Any]: + post = node.get("post", {}) + author = post.get("author", "") + index = post.get("index", "") + time_sent = node.get("post", {}).get("time-sent", 0) + + contents = post.get("contents", []) + text_parts: list[str] = [] + for item in contents: + if "text" in item: + text_parts.append(item["text"]) + elif "mention" in item: + text_parts.append(f"~{item['mention']}") + elif "url" in item: + text_parts.append(item["url"]) + elif "code" in item: + code_block = item["code"] + if isinstance(code_block, dict): + text_parts.append(code_block.get("code", "")) + else: + text_parts.append(str(code_block)) + + return { + "author": author, + "index": index, + "content": "".join(text_parts), + "time": time_sent, + } diff --git a/backend/package/yuxi/channels/adapters/urbit/upload.py b/backend/package/yuxi/channels/adapters/urbit/upload.py new file mode 100644 index 00000000..207d6bef --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/upload.py @@ -0,0 +1,230 @@ +from __future__ import annotations + +import hashlib +import os +import re +import time +from typing import Any, TYPE_CHECKING + +import httpx + +from yuxi.utils.logging_config import logger + +if TYPE_CHECKING: + from .client import UrbitClient + + +_MEMEX_UPLOAD_TIMEOUT = 120.0 +_S3_UPLOAD_TIMEOUT = 300.0 +_UNSAFE_FILENAME_RE = re.compile(r"[/\\:*?\"<>|]") + + +def sanitize_filename(filename: str) -> str: + name = os.path.basename(filename) or "file" + name = _UNSAFE_FILENAME_RE.sub("_", name) + name = name.strip(". ") + if not name: + name = f"file_{hashlib.md5(filename.encode()).hexdigest()[:8]}" + if len(name) > 255: + base, ext = os.path.splitext(name) + name = base[: 255 - len(ext)] + ext + return name + + +async def get_storage_configuration(client: UrbitClient) -> dict[str, Any] | None: + from .scry import scry_get + + result = await scry_get(client, "storage", "/configuration.json") + if isinstance(result, dict): + return result + return None + + +async def get_storage_credentials(client: UrbitClient) -> dict[str, Any] | None: + from .scry import scry_get + + result = await scry_get(client, "storage", "/credentials.json") + if isinstance(result, dict): + return result + return None + + +async def upload_file( + client: UrbitClient, + file_data: bytes, + filename: str, + content_type: str = "application/octet-stream", +) -> str | None: + safe_filename = sanitize_filename(filename) + if safe_filename != filename: + logger.debug(f"[Urbit] Filename sanitized: {filename!r} → {safe_filename!r}") + + storage_config = await get_storage_configuration(client) + if not storage_config: + logger.warning("[Urbit] No storage configuration found, upload skipped") + return None + + backend = storage_config.get("backend", "") + + if backend == "memex" or not backend: + return await _upload_via_memex(client, file_data, safe_filename, content_type) + elif backend == "s3": + return await _upload_via_s3(client, file_data, safe_filename, content_type, storage_config) + else: + logger.warning(f"[Urbit] Unknown storage backend: {backend}") + return None + + +async def _upload_via_memex( + client: UrbitClient, + file_data: bytes, + filename: str, + content_type: str, +) -> str | None: + try: + upload_url = f"{client.ship_url}/~/memex/upload" + files = {"file": (filename, file_data, content_type)} + r = await client.http.post( + upload_url, + files=files, + timeout=httpx.Timeout(_MEMEX_UPLOAD_TIMEOUT), + ) + r.raise_for_status() + result = r.json() + public_url = result.get("url", "") or result.get("publicUrl", "") + if public_url: + logger.info(f"[Urbit] Memex upload success: {filename} → {public_url}") + return public_url + logger.warning("[Urbit] Memex upload response missing URL") + return None + except Exception as e: + logger.warning(f"[Urbit] Memex upload failed for {filename}: {e}") + return None + + +async def _upload_via_s3( + client: UrbitClient, + file_data: bytes, + filename: str, + content_type: str, + storage_config: dict[str, Any], +) -> str | None: + credentials = await get_storage_credentials(client) + if not credentials: + logger.warning("[Urbit] No S3 credentials found") + return None + + endpoint = credentials.get("endpoint", "") + access_key = credentials.get("accessKeyId", "") + secret_key = credentials.get("secretAccessKey", "") + bucket = storage_config.get("bucket", "moltbot-uploads") + + if not all([endpoint, access_key, secret_key]): + logger.warning("[Urbit] Incomplete S3 credentials") + return None + + file_hash = hashlib.sha256(file_data).hexdigest()[:16] + ext = os.path.splitext(filename)[1] or ".bin" + key = f"uploads/{int(time.time())}-{file_hash}{ext}" + + try: + import hmac + from datetime import datetime, UTC + + date_short = datetime.now(UTC).strftime("%Y%m%d") + date_long = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + region = credentials.get("region", "us-east-1") + service = "s3" + + credential = f"{access_key}/{date_short}/{region}/{service}/aws4_request" + + canonical_request = _build_canonical_request("PUT", key, content_type, file_data, bucket) + string_to_sign = _build_string_to_sign(date_long, date_short, region, service, canonical_request) + signing_key = _build_signing_key(secret_key, date_short, region, service) + signature = hmac.new(signing_key, string_to_sign.encode(), "sha256").hexdigest() + + authorization = ( + f"AWS4-HMAC-SHA256 " + f"Credential={credential}, " + f"SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, " + f"Signature={signature}" + ) + + headers = { + "Content-Type": content_type, + "x-amz-content-sha256": hashlib.sha256(file_data).hexdigest(), + "x-amz-date": date_long, + "Authorization": authorization, + } + + upload_url = f"{endpoint}/{bucket}/{key}" + async with httpx.AsyncClient(timeout=httpx.Timeout(_S3_UPLOAD_TIMEOUT)) as s3_client: + r = await s3_client.put(upload_url, content=file_data, headers=headers) + r.raise_for_status() + + public_url = f"{endpoint}/{bucket}/{key}" + logger.info(f"[Urbit] S3 upload success: {filename} → {public_url}") + return public_url + + except ImportError: + logger.warning("[Urbit] S3 upload skipped: hmac module unavailable") + return None + except Exception as e: + logger.warning(f"[Urbit] S3 upload failed for {filename}: {e}") + return None + + +def _build_canonical_request( + method: str, + key: str, + content_type: str, + data: bytes, + bucket: str, +) -> str: + payload_hash = hashlib.sha256(data).hexdigest() + canonical_uri = f"/{key}" + canonical_querystring = "" + canonical_headers = ( + f"content-type:{content_type}\nhost:{bucket}\nx-amz-content-sha256:{payload_hash}\nx-amz-date:{'TIMESTAMP'}\n" + ) + signed_headers = "content-type;host;x-amz-content-sha256;x-amz-date" + return f"{method}\n{canonical_uri}\n{canonical_querystring}\n{canonical_headers}\n{signed_headers}\n{payload_hash}" + + +def _build_string_to_sign( + date_long: str, + date_short: str, + region: str, + service: str, + canonical_request: str, +) -> str: + import hashlib + + scope = f"{date_short}/{region}/{service}/aws4_request" + return f"AWS4-HMAC-SHA256\n{date_long}\n{scope}\n{hashlib.sha256(canonical_request.encode()).hexdigest()}" + + +def _build_signing_key(secret_key: str, date_short: str, region: str, service: str) -> bytes: + import hmac + + k_date = hmac.new(f"AWS4{secret_key}".encode(), date_short.encode(), "sha256").digest() + k_region = hmac.new(k_date, region.encode(), "sha256").digest() + k_service = hmac.new(k_region, service.encode(), "sha256").digest() + return hmac.new(k_service, b"aws4_request", "sha256").digest() + + +def assert_trusted_upload_url(url: str) -> bool: + from .probe import validate_urbit_url + + is_valid, warnings = validate_urbit_url(url) + if not is_valid: + logger.warning(f"[Urbit] Untrusted upload URL: {url} - {warnings}") + return False + return True + + +def assert_safe_upload_result_url(url: str) -> bool: + if not url.startswith(("http://", "https://")): + logger.warning(f"[Urbit] Unsafe upload result URL: {url}") + return False + return True diff --git a/backend/package/yuxi/channels/adapters/urbit/vision.py b/backend/package/yuxi/channels/adapters/urbit/vision.py new file mode 100644 index 00000000..6b819640 --- /dev/null +++ b/backend/package/yuxi/channels/adapters/urbit/vision.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import base64 +from typing import Any, TYPE_CHECKING + +from yuxi.utils.logging_config import logger + +if TYPE_CHECKING: + from .client import UrbitClient + + +async def analyze_image( + client: UrbitClient, + image_url: str, + prompt: str = "Describe this image in detail.", + *, + vision_api_url: str | None = None, + vision_api_key: str | None = None, +) -> str | None: + try: + from .media import download_media + + image_data, content_type = await download_media(client, image_url) + if not image_data: + logger.warning(f"[Urbit] Vision: failed to download image from {image_url}") + return None + + logger.info(f"[Urbit] Vision: downloaded {len(image_data)} bytes ({content_type})") + return await _invoke_vision_pipeline( + image_data, + content_type, + prompt, + vision_api_url=vision_api_url, + vision_api_key=vision_api_key, + ) + + except Exception as e: + logger.warning(f"[Urbit] Vision analysis failed: {e}") + return None + + +async def _invoke_vision_pipeline( + image_data: bytes, + content_type: str, + prompt: str, + *, + vision_api_url: str | None = None, + vision_api_key: str | None = None, +) -> str | None: + if not vision_api_url: + vision_api_url = "https://api.openai.com/v1/chat/completions" + + base64_image = base64.b64encode(image_data).decode("utf-8") + data_url = f"data:{content_type};base64,{base64_image}" + + payload = { + "model": "gpt-4o", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + {"type": "image_url", "image_url": {"url": data_url}}, + ], + } + ], + "max_tokens": 300, + } + + headers: dict[str, str] = {"Content-Type": "application/json"} + if vision_api_key: + headers["Authorization"] = f"Bearer {vision_api_key}" + + import httpx + + async with httpx.AsyncClient(timeout=httpx.Timeout(60.0)) as http: + r = await http.post(vision_api_url, json=payload, headers=headers) + r.raise_for_status() + data = r.json() + choice = data.get("choices", [{}])[0] + message = choice.get("message", {}) + result = message.get("content", "") + if result: + logger.info(f"[Urbit] Vision: analysis complete ({len(result)} chars)") + return result or None + + +async def build_vision_context( + image_urls: list[str], + prompt: str = "What do you see in these images?", +) -> dict[str, Any]: + return { + "images": [{"url": url} for url in image_urls], + "prompt": prompt, + "source": "urbit_vision", + }