Kris
077de8e6d6
实现了完整的网页搜索与抓取内置工具集： 1. 新增搜索工具链：支持SearXNG与Tavily多Provider降级、滑动窗口限流、统一结果格式 2. 新增网页抓取工具：单页抓取、站点递归抓取、内容入库知识库功能 3. 完善安全防护：SSRF校验、robots.txt合规、请求限流 4. 自动注册工具到系统注册表，无需额外配置即可使用
137 lines
4.9 KiB
Python
137 lines
4.9 KiB
Python
"""爬取结果入库：将 Markdown 内容上传 MinIO 并入库到知识库。
|
||
|
||
复用现有 fetch-url + _preprocessed_map 机制：
|
||
1. 将 Markdown 内容上传到 MinIO（参考 knowledge_router.py 的 fetch_url 接口）
|
||
2. 构造 _preprocessed_map，以 source_url 为 key
|
||
3. 调用 KnowledgeBaseManager.add_file_record 创建 UPLOADED 记录
|
||
4. 调用 parse_file 触发解析（Markdown 直接通过，无需 OCR）
|
||
5. 调用 index_file 触发索引
|
||
"""
|
||
import asyncio
|
||
from urllib.parse import urlparse
|
||
|
||
from yuxi.knowledge import knowledge_base
|
||
from yuxi.knowledge.utils.kb_utils import calculate_content_hash
|
||
from yuxi.knowledge.utils.url_fetcher import MAX_DOWNLOAD_SIZE
|
||
from yuxi.repositories.user_repository import UserRepository
|
||
from yuxi.storage.minio.client import MinIOClient, get_minio_client
|
||
from yuxi.utils.logging_config import logger
|
||
|
||
|
||
async def ingest_crawl_result(
|
||
*,
|
||
content: str,
|
||
kb_id: str,
|
||
source_url: str,
|
||
filename: str | None = None,
|
||
operator_id: str,
|
||
) -> dict:
|
||
"""将爬取产出的 Markdown 入库到指定知识库。
|
||
|
||
入库流程复用现有 fetch-url + _preprocessed_map 机制：
|
||
1. 将 Markdown 内容上传到 MinIO（参考 knowledge_router.py 的 fetch_url 接口）
|
||
2. 构造 _preprocessed_map，以 source_url 为 key
|
||
3. 调用 KnowledgeBaseManager.add_file_record 创建 UPLOADED 记录
|
||
4. 调用 parse_file 触发解析（Markdown 直接通过，无需 OCR）
|
||
5. 调用 index_file 触发索引
|
||
|
||
返回 {"file_id": ..., "kb_id": ..., "status": "indexed"} 或抛异常。
|
||
"""
|
||
if not content or not content.strip():
|
||
raise ValueError("爬取内容为空，无法入库")
|
||
|
||
# 大小校验（与 url_fetcher.MAX_DOWNLOAD_SIZE 一致）
|
||
content_bytes = content.encode("utf-8")
|
||
if len(content_bytes) > MAX_DOWNLOAD_SIZE:
|
||
raise ValueError(
|
||
f"爬取内容大小 {len(content_bytes)} 超过限制 {MAX_DOWNLOAD_SIZE}"
|
||
)
|
||
|
||
# 文件名推导：默认从 URL 路径推导，超长截断
|
||
if not filename:
|
||
parsed = urlparse(source_url)
|
||
filename = parsed.path.rstrip("/").split("/")[-1] or parsed.netloc
|
||
if len(filename) > 500:
|
||
filename = filename[:400] + "..." + filename[-90:]
|
||
|
||
# 步骤 1：上传到 MinIO（复用 knowledge_router.py 的 fetch_url 接口模式）
|
||
content_hash = await calculate_content_hash(content_bytes)
|
||
minio_client = get_minio_client()
|
||
bucket_name = MinIOClient.KB_BUCKETS["documents"]
|
||
await asyncio.to_thread(minio_client.ensure_bucket_exists, bucket_name)
|
||
object_name = f"{kb_id}/upload/{content_hash}.md"
|
||
upload_result = await minio_client.aupload_file(
|
||
bucket_name=bucket_name,
|
||
object_name=object_name,
|
||
data=content_bytes,
|
||
content_type="text/markdown",
|
||
)
|
||
minio_url = upload_result.url
|
||
|
||
# 步骤 2：构造 _preprocessed_map
|
||
preprocessed_map = {
|
||
source_url: {
|
||
"filename": filename,
|
||
"path": object_name,
|
||
"content_hash": content_hash,
|
||
"file_size": len(content_bytes),
|
||
}
|
||
}
|
||
|
||
# 步骤 3：创建文件记录（状态为 UPLOADED）
|
||
params = {
|
||
"_preprocessed_map": preprocessed_map,
|
||
"content_type": "file",
|
||
}
|
||
record = await knowledge_base.add_file_record(
|
||
kb_id=kb_id,
|
||
item=source_url,
|
||
params=params,
|
||
operator_id=operator_id,
|
||
)
|
||
file_id = record["file_id"]
|
||
|
||
# 步骤 4：触发解析（Markdown 直接通过，无需 OCR）
|
||
await knowledge_base.parse_file(
|
||
kb_id=kb_id, file_id=file_id, operator_id=operator_id
|
||
)
|
||
|
||
# 步骤 5：触发索引
|
||
await knowledge_base.index_file(
|
||
kb_id=kb_id, file_id=file_id, operator_id=operator_id
|
||
)
|
||
|
||
logger.info(
|
||
f"crawl_ingest_success kb_id={kb_id} file_id={file_id} source_url={source_url}"
|
||
)
|
||
return {"file_id": file_id, "kb_id": kb_id, "status": "indexed", "minio_url": minio_url}
|
||
|
||
|
||
async def resolve_user_dict(uid: str) -> dict:
|
||
"""通过 uid 查询用户信息，返回 check_accessible 所需的 user dict。
|
||
|
||
KnowledgeBaseManager.check_accessible(user, kb_id) 接受 user dict，
|
||
包含 uid / role / department_id 字段。BaseContext 仅暴露 uid，
|
||
故需查询 UserRepository 补全 role 与 department_id。
|
||
"""
|
||
if not uid:
|
||
return {"uid": "", "role": "", "department_id": None}
|
||
user_repo = UserRepository()
|
||
user = await user_repo.get_by_uid(uid)
|
||
if user is None:
|
||
return {"uid": uid, "role": "", "department_id": None}
|
||
return {
|
||
"uid": str(user.uid),
|
||
"role": user.role,
|
||
"department_id": user.department_id,
|
||
}
|
||
|
||
|
||
async def check_kb_permission(kb_id: str, user: dict) -> bool:
|
||
"""校验用户对目标知识库的写入权限。
|
||
|
||
复用 KnowledgeBaseManager.check_accessible(user, kb_id)，
|
||
覆盖 share_config 三级共享（global / department / user）与 superadmin 直通。
|
||
"""
|
||
return await knowledge_base.check_accessible(user, kb_id)
|