113 lines
3.7 KiB
Python
113 lines
3.7 KiB
Python
|
"""Crawl4AI 服务 HTTP 客户端，封装 /crawl 端点调用。
|
|||
|
|
|
|||
|
|
Crawl4AI 服务监听 11235 端口（Gunicorn），仅内网访问 http://crawl4ai:11235。
|
|||
|
|
本模块负责单页抓取与多页递归抓取的 HTTP 调用封装。
|
|||
|
|
"""
|
|||
|
|
import httpx
|
|||
|
|
|
|||
|
|
from yuxi.config import config
|
|||
|
|
|
|||
|
|
|
|||
|
|
class Crawl4AIClient:
|
|||
|
|
"""Crawl4AI 服务 HTTP 客户端，封装 /crawl 端点调用。
|
|||
|
|
|
|||
|
|
Crawl4AI 服务监听 11235 端口（Gunicorn），仅内网访问 http://crawl4ai:11235。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(self):
|
|||
|
|
self._client: httpx.AsyncClient | None = None
|
|||
|
|
|
|||
|
|
async def crawl_single(
|
|||
|
|
self,
|
|||
|
|
url: str,
|
|||
|
|
*,
|
|||
|
|
output_format: str = "markdown",
|
|||
|
|
check_robots_txt: bool = True,
|
|||
|
|
) -> dict:
|
|||
|
|
"""单页抓取，返回 CrawlResult JSON。
|
|||
|
|
|
|||
|
|
Crawl4AI /crawl 接口请求体字段为 urls（复数，可传字符串或数组）。
|
|||
|
|
"""
|
|||
|
|
base_url = config.crawl4ai_base_url
|
|||
|
|
timeout = config.crawl4ai_timeout
|
|||
|
|
headers = self._build_headers()
|
|||
|
|
body = {
|
|||
|
|
"urls": url,
|
|||
|
|
"crawler_run_config": {
|
|||
|
|
"output_format": output_format,
|
|||
|
|
"check_robots_txt": check_robots_txt,
|
|||
|
|
"fit_markdown": True, # 启用 fit_markdown 噪声剥离
|
|||
|
|
},
|
|||
|
|
}
|
|||
|
|
client = self._get_client()
|
|||
|
|
response = await client.post(
|
|||
|
|
f"{base_url}/crawl", json=body, headers=headers, timeout=timeout
|
|||
|
|
)
|
|||
|
|
response.raise_for_status()
|
|||
|
|
data = response.json()
|
|||
|
|
# Crawl4AI 返回结构为 {"results": [CrawlResult, ...]}
|
|||
|
|
results = data.get("results", [])
|
|||
|
|
return results[0] if results else {}
|
|||
|
|
|
|||
|
|
async def crawl_many(
|
|||
|
|
self,
|
|||
|
|
urls: list[str],
|
|||
|
|
*,
|
|||
|
|
max_depth: int = 2,
|
|||
|
|
max_pages: int = 20,
|
|||
|
|
url_pattern: str = "",
|
|||
|
|
check_robots_txt: bool = True,
|
|||
|
|
semaphore_count: int = 5,
|
|||
|
|
) -> list[dict]:
|
|||
|
|
"""多页递归抓取，返回 CrawlResult 列表。
|
|||
|
|
|
|||
|
|
使用 Crawl4AI BFSDeepCrawlStrategy（广度优先），include_external=False 仅抓同域。
|
|||
|
|
"""
|
|||
|
|
base_url = config.crawl4ai_base_url
|
|||
|
|
# 多页抓取超时放宽到 5 分钟
|
|||
|
|
timeout = min(300, config.crawl4ai_timeout * 10)
|
|||
|
|
headers = self._build_headers()
|
|||
|
|
crawler_config = {
|
|||
|
|
"deep_crawl_strategy": {
|
|||
|
|
"type": "BFSDeepCrawlStrategy",
|
|||
|
|
"max_depth": max_depth,
|
|||
|
|
"max_pages": max_pages,
|
|||
|
|
"include_external": False,
|
|||
|
|
"url_pattern": url_pattern or None,
|
|||
|
|
},
|
|||
|
|
"crawler_run_config": {
|
|||
|
|
"output_format": "markdown",
|
|||
|
|
"check_robots_txt": check_robots_txt,
|
|||
|
|
"fit_markdown": True,
|
|||
|
|
},
|
|||
|
|
"arun_many_config": {
|
|||
|
|
"semaphore_count": semaphore_count,
|
|||
|
|
},
|
|||
|
|
}
|
|||
|
|
body = {"urls": urls, **crawler_config}
|
|||
|
|
client = self._get_client()
|
|||
|
|
response = await client.post(
|
|||
|
|
f"{base_url}/crawl", json=body, headers=headers, timeout=timeout
|
|||
|
|
)
|
|||
|
|
response.raise_for_status()
|
|||
|
|
data = response.json()
|
|||
|
|
return data.get("results", [])
|
|||
|
|
|
|||
|
|
def _build_headers(self) -> dict:
|
|||
|
|
"""构建请求头，含可选的 Bearer Token 鉴权。"""
|
|||
|
|
headers = {"Content-Type": "application/json"}
|
|||
|
|
token = config.crawl4ai_api_token
|
|||
|
|
if token:
|
|||
|
|
headers["Authorization"] = f"Bearer {token}"
|
|||
|
|
return headers
|
|||
|
|
|
|||
|
|
def _get_client(self) -> httpx.AsyncClient:
|
|||
|
|
"""懒初始化并复用 httpx.AsyncClient 连接池。"""
|
|||
|
|
if self._client is None or self._client.is_closed:
|
|||
|
|
self._client = httpx.AsyncClient()
|
|||
|
|
return self._client
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 模块级单例
|
|||
|
|
crawl4ai_client = Crawl4AIClient()
|