Kris
077de8e6d6
实现了完整的网页搜索与抓取内置工具集： 1. 新增搜索工具链：支持SearXNG与Tavily多Provider降级、滑动窗口限流、统一结果格式 2. 新增网页抓取工具：单页抓取、站点递归抓取、内容入库知识库功能 3. 完善安全防护：SSRF校验、robots.txt合规、请求限流 4. 自动注册工具到系统注册表，无需额外配置即可使用
113 lines
3.7 KiB
Python
113 lines
3.7 KiB
Python
"""Crawl4AI 服务 HTTP 客户端，封装 /crawl 端点调用。
|
||
|
||
Crawl4AI 服务监听 11235 端口（Gunicorn），仅内网访问 http://crawl4ai:11235。
|
||
本模块负责单页抓取与多页递归抓取的 HTTP 调用封装。
|
||
"""
|
||
import httpx
|
||
|
||
from yuxi.config import config
|
||
|
||
|
||
class Crawl4AIClient:
|
||
"""Crawl4AI 服务 HTTP 客户端，封装 /crawl 端点调用。
|
||
|
||
Crawl4AI 服务监听 11235 端口（Gunicorn），仅内网访问 http://crawl4ai:11235。
|
||
"""
|
||
|
||
def __init__(self):
|
||
self._client: httpx.AsyncClient | None = None
|
||
|
||
async def crawl_single(
|
||
self,
|
||
url: str,
|
||
*,
|
||
output_format: str = "markdown",
|
||
check_robots_txt: bool = True,
|
||
) -> dict:
|
||
"""单页抓取，返回 CrawlResult JSON。
|
||
|
||
Crawl4AI /crawl 接口请求体字段为 urls（复数，可传字符串或数组）。
|
||
"""
|
||
base_url = config.crawl4ai_base_url
|
||
timeout = config.crawl4ai_timeout
|
||
headers = self._build_headers()
|
||
body = {
|
||
"urls": url,
|
||
"crawler_run_config": {
|
||
"output_format": output_format,
|
||
"check_robots_txt": check_robots_txt,
|
||
"fit_markdown": True, # 启用 fit_markdown 噪声剥离
|
||
},
|
||
}
|
||
client = self._get_client()
|
||
response = await client.post(
|
||
f"{base_url}/crawl", json=body, headers=headers, timeout=timeout
|
||
)
|
||
response.raise_for_status()
|
||
data = response.json()
|
||
# Crawl4AI 返回结构为 {"results": [CrawlResult, ...]}
|
||
results = data.get("results", [])
|
||
return results[0] if results else {}
|
||
|
||
async def crawl_many(
|
||
self,
|
||
urls: list[str],
|
||
*,
|
||
max_depth: int = 2,
|
||
max_pages: int = 20,
|
||
url_pattern: str = "",
|
||
check_robots_txt: bool = True,
|
||
semaphore_count: int = 5,
|
||
) -> list[dict]:
|
||
"""多页递归抓取，返回 CrawlResult 列表。
|
||
|
||
使用 Crawl4AI BFSDeepCrawlStrategy（广度优先），include_external=False 仅抓同域。
|
||
"""
|
||
base_url = config.crawl4ai_base_url
|
||
# 多页抓取超时放宽到 5 分钟
|
||
timeout = min(300, config.crawl4ai_timeout * 10)
|
||
headers = self._build_headers()
|
||
crawler_config = {
|
||
"deep_crawl_strategy": {
|
||
"type": "BFSDeepCrawlStrategy",
|
||
"max_depth": max_depth,
|
||
"max_pages": max_pages,
|
||
"include_external": False,
|
||
"url_pattern": url_pattern or None,
|
||
},
|
||
"crawler_run_config": {
|
||
"output_format": "markdown",
|
||
"check_robots_txt": check_robots_txt,
|
||
"fit_markdown": True,
|
||
},
|
||
"arun_many_config": {
|
||
"semaphore_count": semaphore_count,
|
||
},
|
||
}
|
||
body = {"urls": urls, **crawler_config}
|
||
client = self._get_client()
|
||
response = await client.post(
|
||
f"{base_url}/crawl", json=body, headers=headers, timeout=timeout
|
||
)
|
||
response.raise_for_status()
|
||
data = response.json()
|
||
return data.get("results", [])
|
||
|
||
def _build_headers(self) -> dict:
|
||
"""构建请求头，含可选的 Bearer Token 鉴权。"""
|
||
headers = {"Content-Type": "application/json"}
|
||
token = config.crawl4ai_api_token
|
||
if token:
|
||
headers["Authorization"] = f"Bearer {token}"
|
||
return headers
|
||
|
||
def _get_client(self) -> httpx.AsyncClient:
|
||
"""懒初始化并复用 httpx.AsyncClient 连接池。"""
|
||
if self._client is None or self._client.is_closed:
|
||
self._client = httpx.AsyncClient()
|
||
return self._client
|
||
|
||
|
||
# 模块级单例
|
||
crawl4ai_client = Crawl4AIClient()
|