256 lines
8.4 KiB
Python
256 lines
8.4 KiB
Python
|
import asyncio
|
||
|
|
import logging
|
||
|
|
import time
|
||
|
|
|
||
|
|
import httpx
|
||
|
|
|
||
|
|
from yuxi.channel.extensions.clickup.dedupe import MessageDeduplicator
|
||
|
|
from yuxi.channel.extensions.clickup.types import ClickUpAccount
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
API_V3_BASE = "https://api.clickup.com/api/v3"
|
||
|
|
|
||
|
|
|
||
|
|
class ClickUpPolling:
|
||
|
|
def __init__(self):
|
||
|
|
self._running = False
|
||
|
|
self._task: asyncio.Task | None = None
|
||
|
|
self._last_polled_at: float = 0.0
|
||
|
|
self._deduplicator = MessageDeduplicator(max_size=10000, ttl_seconds=300)
|
||
|
|
self._cursor: str | None = None
|
||
|
|
|
||
|
|
async def start(
|
||
|
|
self,
|
||
|
|
account: ClickUpAccount,
|
||
|
|
queue: asyncio.Queue,
|
||
|
|
abort_event: asyncio.Event,
|
||
|
|
) -> None:
|
||
|
|
if self._running:
|
||
|
|
return
|
||
|
|
|
||
|
|
self._running = True
|
||
|
|
self._last_polled_at = time.time()
|
||
|
|
self._task = asyncio.create_task(self._polling_loop(account, queue, abort_event))
|
||
|
|
logger.info("clickup polling started for account %s, interval=%.1fs", account.account_id, account.poll_interval)
|
||
|
|
|
||
|
|
async def stop(self) -> None:
|
||
|
|
self._running = False
|
||
|
|
if self._task:
|
||
|
|
self._task.cancel()
|
||
|
|
try:
|
||
|
|
await self._task
|
||
|
|
except asyncio.CancelledError:
|
||
|
|
pass
|
||
|
|
self._task = None
|
||
|
|
logger.info("clickup polling stopped")
|
||
|
|
|
||
|
|
async def _polling_loop(
|
||
|
|
self,
|
||
|
|
account: ClickUpAccount,
|
||
|
|
queue: asyncio.Queue,
|
||
|
|
abort_event: asyncio.Event,
|
||
|
|
) -> None:
|
||
|
|
backoff = 1
|
||
|
|
max_backoff = 300
|
||
|
|
interval = account.poll_interval
|
||
|
|
|
||
|
|
while self._running and not abort_event.is_set():
|
||
|
|
try:
|
||
|
|
new_messages = await self._fetch_recent_messages(account)
|
||
|
|
for msg in new_messages:
|
||
|
|
if queue.qsize() < queue.maxsize:
|
||
|
|
await queue.put(msg)
|
||
|
|
backoff = 1
|
||
|
|
except asyncio.CancelledError:
|
||
|
|
break
|
||
|
|
except Exception:
|
||
|
|
logger.exception("clickup polling error, backoff=%ds", backoff)
|
||
|
|
await asyncio.sleep(backoff)
|
||
|
|
backoff = min(backoff * 2, max_backoff)
|
||
|
|
continue
|
||
|
|
|
||
|
|
try:
|
||
|
|
await asyncio.wait_for(abort_event.wait(), timeout=interval)
|
||
|
|
break
|
||
|
|
except TimeoutError:
|
||
|
|
continue
|
||
|
|
|
||
|
|
async def _fetch_recent_messages(self, account: ClickUpAccount) -> list[dict]:
|
||
|
|
headers = {"Authorization": account.api_token}
|
||
|
|
results: list[dict] = []
|
||
|
|
|
||
|
|
async with httpx.AsyncClient(timeout=30.0) as client:
|
||
|
|
channel_messages = await self._poll_channel_messages(account, client, headers)
|
||
|
|
results.extend(channel_messages)
|
||
|
|
|
||
|
|
dm_messages = await self._poll_dm_messages(account, client, headers)
|
||
|
|
results.extend(dm_messages)
|
||
|
|
|
||
|
|
self._last_polled_at = time.time()
|
||
|
|
return results
|
||
|
|
|
||
|
|
async def _poll_channel_messages(
|
||
|
|
self,
|
||
|
|
account: ClickUpAccount,
|
||
|
|
client: httpx.AsyncClient,
|
||
|
|
headers: dict,
|
||
|
|
) -> list[dict]:
|
||
|
|
try:
|
||
|
|
channels = await self._list_channels(account, client, headers)
|
||
|
|
except Exception:
|
||
|
|
logger.exception("clickup polling: failed to list channels")
|
||
|
|
return []
|
||
|
|
|
||
|
|
results: list[dict] = []
|
||
|
|
for channel in channels:
|
||
|
|
channel_id = channel.get("id", "")
|
||
|
|
if not channel_id:
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
messages = await self._fetch_channel_messages(account, client, headers, channel_id)
|
||
|
|
results.extend(messages)
|
||
|
|
except Exception:
|
||
|
|
logger.exception("clickup polling: failed to fetch messages for channel %s", channel_id)
|
||
|
|
continue
|
||
|
|
|
||
|
|
return results
|
||
|
|
|
||
|
|
async def _poll_dm_messages(
|
||
|
|
self,
|
||
|
|
account: ClickUpAccount,
|
||
|
|
client: httpx.AsyncClient,
|
||
|
|
headers: dict,
|
||
|
|
) -> list[dict]:
|
||
|
|
url = f"{API_V3_BASE}/workspaces/{account.workspace_id}/chat/dm"
|
||
|
|
try:
|
||
|
|
resp = await client.get(url, headers=headers)
|
||
|
|
if resp.status_code != 200:
|
||
|
|
return []
|
||
|
|
data = resp.json()
|
||
|
|
dms = data.get("channels", [])
|
||
|
|
except Exception:
|
||
|
|
logger.exception("clickup polling: failed to list dms")
|
||
|
|
return []
|
||
|
|
|
||
|
|
results: list[dict] = []
|
||
|
|
for dm in dms:
|
||
|
|
dm_id = dm.get("id", "")
|
||
|
|
if not dm_id:
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
messages = await self._fetch_dm_messages(account, client, headers, dm_id)
|
||
|
|
for msg in messages:
|
||
|
|
msg["chat_type"] = "direct"
|
||
|
|
results.extend(messages)
|
||
|
|
except Exception:
|
||
|
|
logger.exception("clickup polling: failed to fetch messages for dm %s", dm_id)
|
||
|
|
continue
|
||
|
|
|
||
|
|
return results
|
||
|
|
|
||
|
|
async def _list_channels(
|
||
|
|
self,
|
||
|
|
account: ClickUpAccount,
|
||
|
|
client: httpx.AsyncClient,
|
||
|
|
headers: dict,
|
||
|
|
) -> list[dict]:
|
||
|
|
url = f"{API_V3_BASE}/workspaces/{account.workspace_id}/chat/channels"
|
||
|
|
resp = await client.get(url, headers=headers)
|
||
|
|
if resp.status_code != 200:
|
||
|
|
logger.warning("clickup polling: list channels failed status=%d", resp.status_code)
|
||
|
|
return []
|
||
|
|
data = resp.json()
|
||
|
|
return data.get("channels", [])
|
||
|
|
|
||
|
|
async def _fetch_channel_messages(
|
||
|
|
self,
|
||
|
|
account: ClickUpAccount,
|
||
|
|
client: httpx.AsyncClient,
|
||
|
|
headers: dict,
|
||
|
|
channel_id: str,
|
||
|
|
) -> list[dict]:
|
||
|
|
url = f"{API_V3_BASE}/workspaces/{account.workspace_id}/chat/channels/{channel_id}/messages"
|
||
|
|
return await self._fetch_messages_with_cursor(account, client, headers, url, channel_id, "group")
|
||
|
|
|
||
|
|
async def _fetch_dm_messages(
|
||
|
|
self,
|
||
|
|
account: ClickUpAccount,
|
||
|
|
client: httpx.AsyncClient,
|
||
|
|
headers: dict,
|
||
|
|
dm_id: str,
|
||
|
|
) -> list[dict]:
|
||
|
|
url = f"{API_V3_BASE}/workspaces/{account.workspace_id}/chat/dm/{dm_id}/messages"
|
||
|
|
return await self._fetch_messages_with_cursor(account, client, headers, url, dm_id, "direct")
|
||
|
|
|
||
|
|
async def _fetch_messages_with_cursor(
|
||
|
|
self,
|
||
|
|
account: ClickUpAccount,
|
||
|
|
client: httpx.AsyncClient,
|
||
|
|
headers: dict,
|
||
|
|
url: str,
|
||
|
|
chat_id: str,
|
||
|
|
chat_type: str,
|
||
|
|
) -> list[dict]:
|
||
|
|
results: list[dict] = []
|
||
|
|
cursor: str | None = None
|
||
|
|
seen_ids: set = set()
|
||
|
|
|
||
|
|
while True:
|
||
|
|
params: dict = {"limit": 50}
|
||
|
|
if cursor:
|
||
|
|
params["cursor"] = cursor
|
||
|
|
|
||
|
|
resp = await client.get(url, params=params, headers=headers)
|
||
|
|
if resp.status_code != 200:
|
||
|
|
break
|
||
|
|
|
||
|
|
data = resp.json()
|
||
|
|
raw_messages = data.get("messages", [])
|
||
|
|
hit_old = False
|
||
|
|
|
||
|
|
for msg in raw_messages:
|
||
|
|
msg_id = msg.get("id", "")
|
||
|
|
if not msg_id or msg_id in seen_ids:
|
||
|
|
continue
|
||
|
|
seen_ids.add(msg_id)
|
||
|
|
if self._deduplicator.is_duplicate(msg_id):
|
||
|
|
continue
|
||
|
|
|
||
|
|
created_ts = msg.get("date_created", "")
|
||
|
|
if created_ts:
|
||
|
|
try:
|
||
|
|
created_at = float(created_ts) / 1000.0 if len(str(created_ts)) > 10 else float(created_ts)
|
||
|
|
except (ValueError, TypeError):
|
||
|
|
created_at = 0.0
|
||
|
|
if created_at < self._last_polled_at - 5:
|
||
|
|
hit_old = True
|
||
|
|
break
|
||
|
|
|
||
|
|
user = msg.get("user", {})
|
||
|
|
unified = {
|
||
|
|
"channel": "clickup",
|
||
|
|
"msg_id": msg_id,
|
||
|
|
"chat_id": chat_id,
|
||
|
|
"chat_type": chat_type,
|
||
|
|
"sender": {
|
||
|
|
"id": str(user.get("id", "")),
|
||
|
|
"name": user.get("name", ""),
|
||
|
|
},
|
||
|
|
"text": msg.get("content", ""),
|
||
|
|
"content_format": msg.get("content_format", "text/md"),
|
||
|
|
"reply_to_id": msg.get("reply_to_id"),
|
||
|
|
"timestamp": created_ts if isinstance(created_ts, str) else None,
|
||
|
|
"raw": msg,
|
||
|
|
}
|
||
|
|
results.append(unified)
|
||
|
|
|
||
|
|
if hit_old:
|
||
|
|
break
|
||
|
|
|
||
|
|
cursor = data.get("cursor")
|
||
|
|
if not cursor:
|
||
|
|
break
|
||
|
|
|
||
|
|
return results
|