"""爬取结果入库:将 Markdown 内容上传 MinIO 并入库到知识库。 复用现有 fetch-url + _preprocessed_map 机制: 1. 将 Markdown 内容上传到 MinIO(参考 knowledge_router.py 的 fetch_url 接口) 2. 构造 _preprocessed_map,以 source_url 为 key 3. 调用 KnowledgeBaseManager.add_file_record 创建 UPLOADED 记录 4. 调用 parse_file 触发解析(Markdown 直接通过,无需 OCR) 5. 调用 index_file 触发索引 """ import asyncio from urllib.parse import urlparse from yuxi.knowledge import knowledge_base from yuxi.knowledge.utils.kb_utils import calculate_content_hash from yuxi.knowledge.utils.url_fetcher import MAX_DOWNLOAD_SIZE from yuxi.repositories.user_repository import UserRepository from yuxi.storage.minio.client import MinIOClient, get_minio_client from yuxi.utils.logging_config import logger async def ingest_crawl_result( *, content: str, kb_id: str, source_url: str, filename: str | None = None, operator_id: str, ) -> dict: """将爬取产出的 Markdown 入库到指定知识库。 入库流程复用现有 fetch-url + _preprocessed_map 机制: 1. 将 Markdown 内容上传到 MinIO(参考 knowledge_router.py 的 fetch_url 接口) 2. 构造 _preprocessed_map,以 source_url 为 key 3. 调用 KnowledgeBaseManager.add_file_record 创建 UPLOADED 记录 4. 调用 parse_file 触发解析(Markdown 直接通过,无需 OCR) 5. 调用 index_file 触发索引 返回 {"file_id": ..., "kb_id": ..., "status": "indexed"} 或抛异常。 """ if not content or not content.strip(): raise ValueError("爬取内容为空,无法入库") # 大小校验(与 url_fetcher.MAX_DOWNLOAD_SIZE 一致) content_bytes = content.encode("utf-8") if len(content_bytes) > MAX_DOWNLOAD_SIZE: raise ValueError( f"爬取内容大小 {len(content_bytes)} 超过限制 {MAX_DOWNLOAD_SIZE}" ) # 文件名推导:默认从 URL 路径推导,超长截断 if not filename: parsed = urlparse(source_url) filename = parsed.path.rstrip("/").split("/")[-1] or parsed.netloc if len(filename) > 500: filename = filename[:400] + "..." + filename[-90:] # 步骤 1:上传到 MinIO(复用 knowledge_router.py 的 fetch_url 接口模式) content_hash = await calculate_content_hash(content_bytes) minio_client = get_minio_client() bucket_name = MinIOClient.KB_BUCKETS["documents"] await asyncio.to_thread(minio_client.ensure_bucket_exists, bucket_name) object_name = f"{kb_id}/upload/{content_hash}.md" upload_result = await minio_client.aupload_file( bucket_name=bucket_name, object_name=object_name, data=content_bytes, content_type="text/markdown", ) minio_url = upload_result.url # 步骤 2:构造 _preprocessed_map preprocessed_map = { source_url: { "filename": filename, "path": object_name, "content_hash": content_hash, "file_size": len(content_bytes), } } # 步骤 3:创建文件记录(状态为 UPLOADED) params = { "_preprocessed_map": preprocessed_map, "content_type": "file", } record = await knowledge_base.add_file_record( kb_id=kb_id, item=source_url, params=params, operator_id=operator_id, ) file_id = record["file_id"] # 步骤 4:触发解析(Markdown 直接通过,无需 OCR) await knowledge_base.parse_file( kb_id=kb_id, file_id=file_id, operator_id=operator_id ) # 步骤 5:触发索引 await knowledge_base.index_file( kb_id=kb_id, file_id=file_id, operator_id=operator_id ) logger.info( f"crawl_ingest_success kb_id={kb_id} file_id={file_id} source_url={source_url}" ) return {"file_id": file_id, "kb_id": kb_id, "status": "indexed", "minio_url": minio_url} async def resolve_user_dict(uid: str) -> dict: """通过 uid 查询用户信息,返回 check_accessible 所需的 user dict。 KnowledgeBaseManager.check_accessible(user, kb_id) 接受 user dict, 包含 uid / role / department_id 字段。BaseContext 仅暴露 uid, 故需查询 UserRepository 补全 role 与 department_id。 """ if not uid: return {"uid": "", "role": "", "department_id": None} user_repo = UserRepository() user = await user_repo.get_by_uid(uid) if user is None: return {"uid": uid, "role": "", "department_id": None} return { "uid": str(user.uid), "role": user.role, "department_id": user.department_id, } async def check_kb_permission(kb_id: str, user: dict) -> bool: """校验用户对目标知识库的写入权限。 复用 KnowledgeBaseManager.check_accessible(user, kb_id), 覆盖 share_config 三级共享(global / department / user)与 superadmin 直通。 """ return await knowledge_base.check_accessible(user, kb_id)