import asyncio import logging import time import httpx from yuxi.channel.extensions.clickup.dedupe import MessageDeduplicator from yuxi.channel.extensions.clickup.types import ClickUpAccount logger = logging.getLogger(__name__) API_V3_BASE = "https://api.clickup.com/api/v3" class ClickUpPolling: def __init__(self): self._running = False self._task: asyncio.Task | None = None self._last_polled_at: float = 0.0 self._deduplicator = MessageDeduplicator(max_size=10000, ttl_seconds=300) self._cursor: str | None = None async def start( self, account: ClickUpAccount, queue: asyncio.Queue, abort_event: asyncio.Event, ) -> None: if self._running: return self._running = True self._last_polled_at = time.time() self._task = asyncio.create_task(self._polling_loop(account, queue, abort_event)) logger.info("clickup polling started for account %s, interval=%.1fs", account.account_id, account.poll_interval) async def stop(self) -> None: self._running = False if self._task: self._task.cancel() try: await self._task except asyncio.CancelledError: pass self._task = None logger.info("clickup polling stopped") async def _polling_loop( self, account: ClickUpAccount, queue: asyncio.Queue, abort_event: asyncio.Event, ) -> None: backoff = 1 max_backoff = 300 interval = account.poll_interval while self._running and not abort_event.is_set(): try: new_messages = await self._fetch_recent_messages(account) for msg in new_messages: if queue.qsize() < queue.maxsize: await queue.put(msg) backoff = 1 except asyncio.CancelledError: break except Exception: logger.exception("clickup polling error, backoff=%ds", backoff) await asyncio.sleep(backoff) backoff = min(backoff * 2, max_backoff) continue try: await asyncio.wait_for(abort_event.wait(), timeout=interval) break except TimeoutError: continue async def _fetch_recent_messages(self, account: ClickUpAccount) -> list[dict]: headers = {"Authorization": account.api_token} results: list[dict] = [] async with httpx.AsyncClient(timeout=30.0) as client: channel_messages = await self._poll_channel_messages(account, client, headers) results.extend(channel_messages) dm_messages = await self._poll_dm_messages(account, client, headers) results.extend(dm_messages) self._last_polled_at = time.time() return results async def _poll_channel_messages( self, account: ClickUpAccount, client: httpx.AsyncClient, headers: dict, ) -> list[dict]: try: channels = await self._list_channels(account, client, headers) except Exception: logger.exception("clickup polling: failed to list channels") return [] results: list[dict] = [] for channel in channels: channel_id = channel.get("id", "") if not channel_id: continue try: messages = await self._fetch_channel_messages(account, client, headers, channel_id) results.extend(messages) except Exception: logger.exception("clickup polling: failed to fetch messages for channel %s", channel_id) continue return results async def _poll_dm_messages( self, account: ClickUpAccount, client: httpx.AsyncClient, headers: dict, ) -> list[dict]: url = f"{API_V3_BASE}/workspaces/{account.workspace_id}/chat/dm" try: resp = await client.get(url, headers=headers) if resp.status_code != 200: return [] data = resp.json() dms = data.get("channels", []) except Exception: logger.exception("clickup polling: failed to list dms") return [] results: list[dict] = [] for dm in dms: dm_id = dm.get("id", "") if not dm_id: continue try: messages = await self._fetch_dm_messages(account, client, headers, dm_id) for msg in messages: msg["chat_type"] = "direct" results.extend(messages) except Exception: logger.exception("clickup polling: failed to fetch messages for dm %s", dm_id) continue return results async def _list_channels( self, account: ClickUpAccount, client: httpx.AsyncClient, headers: dict, ) -> list[dict]: url = f"{API_V3_BASE}/workspaces/{account.workspace_id}/chat/channels" resp = await client.get(url, headers=headers) if resp.status_code != 200: logger.warning("clickup polling: list channels failed status=%d", resp.status_code) return [] data = resp.json() return data.get("channels", []) async def _fetch_channel_messages( self, account: ClickUpAccount, client: httpx.AsyncClient, headers: dict, channel_id: str, ) -> list[dict]: url = f"{API_V3_BASE}/workspaces/{account.workspace_id}/chat/channels/{channel_id}/messages" return await self._fetch_messages_with_cursor(account, client, headers, url, channel_id, "group") async def _fetch_dm_messages( self, account: ClickUpAccount, client: httpx.AsyncClient, headers: dict, dm_id: str, ) -> list[dict]: url = f"{API_V3_BASE}/workspaces/{account.workspace_id}/chat/dm/{dm_id}/messages" return await self._fetch_messages_with_cursor(account, client, headers, url, dm_id, "direct") async def _fetch_messages_with_cursor( self, account: ClickUpAccount, client: httpx.AsyncClient, headers: dict, url: str, chat_id: str, chat_type: str, ) -> list[dict]: results: list[dict] = [] cursor: str | None = None seen_ids: set = set() while True: params: dict = {"limit": 50} if cursor: params["cursor"] = cursor resp = await client.get(url, params=params, headers=headers) if resp.status_code != 200: break data = resp.json() raw_messages = data.get("messages", []) hit_old = False for msg in raw_messages: msg_id = msg.get("id", "") if not msg_id or msg_id in seen_ids: continue seen_ids.add(msg_id) if self._deduplicator.is_duplicate(msg_id): continue created_ts = msg.get("date_created", "") if created_ts: try: created_at = float(created_ts) / 1000.0 if len(str(created_ts)) > 10 else float(created_ts) except (ValueError, TypeError): created_at = 0.0 if created_at < self._last_polled_at - 5: hit_old = True break user = msg.get("user", {}) unified = { "channel": "clickup", "msg_id": msg_id, "chat_id": chat_id, "chat_type": chat_type, "sender": { "id": str(user.get("id", "")), "name": user.get("name", ""), }, "text": msg.get("content", ""), "content_format": msg.get("content_format", "text/md"), "reply_to_id": msg.get("reply_to_id"), "timestamp": created_ts if isinstance(created_ts, str) else None, "raw": msg, } results.append(unified) if hit_old: break cursor = data.get("cursor") if not cursor: break return results