From 0875e9769c8bcb10ac7737a9d69440a1c81201d6 Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Sun, 14 Dec 2025 18:13:46 +0800 Subject: [PATCH] =?UTF-8?q?feat(eval):=20=E6=B7=BB=E5=8A=A0=E8=AF=84?= =?UTF-8?q?=E4=BC=B0=E5=9F=BA=E5=87=86=E5=88=86=E9=A1=B5=E5=92=8C=E9=94=99?= =?UTF-8?q?=E8=AF=AF=E7=BB=93=E6=9E=9C=E8=BF=87=E6=BB=A4=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 实现评估基准和结果的分页查询，支持按错误过滤结果 优化评估结果表格展示，增加文档链接和说明 提升文件上传验证逻辑和错误处理 --- docs/latest/changelog/roadmap.md | 2 +- docs/latest/intro/evaluation.md | 2 + server/routers/evaluation_router.py | 39 ++- src/services/evaluation_service.py | 109 ++++++- web/src/apis/knowledge_api.js | 19 +- web/src/components/EvaluationBenchmarks.vue | 272 ++++++++++++++---- web/src/components/FileUploadModal.vue | 12 + web/src/components/RAGEvaluationTab.vue | 265 +++++++++++++---- .../modals/BenchmarkGenerateModal.vue | 14 +- .../modals/BenchmarkUploadModal.vue | 85 +++--- 10 files changed, 657 insertions(+), 162 deletions(-) diff --git a/docs/latest/changelog/roadmap.md b/docs/latest/changelog/roadmap.md index 6ea3c8bd..ae6be778 100644 --- a/docs/latest/changelog/roadmap.md +++ b/docs/latest/changelog/roadmap.md @@ -15,7 +15,7 @@ - 同名文件处理逻辑：遇到同名文件则在上传区域提示，是否删除旧文件 - conversation 待修改为异步的版本 - DBManager 需要将数据库修改为异步的aiosqlite或者异步mysql，缓存使用Redis存储 -- 【eval】缺少自动生成评估的功能 +- 【eval】支持 easy dataset 的数据格式 - chat_model 的 call 需要异步 ### Bugs diff --git a/docs/latest/intro/evaluation.md b/docs/latest/intro/evaluation.md index 216d36b1..ec2f35ea 100644 --- a/docs/latest/intro/evaluation.md +++ b/docs/latest/intro/evaluation.md @@ -16,6 +16,8 @@ Yuxi-Know 提供了完整的 RAG 系统评估解决方案，帮助您科学地 - 当基准包含标准答案时，系统会使用 LLM 作为评判者评估生成答案的准确性 - 同时包含两者时，系统会进行全面的检索和生成质量评估 +也可以从 easy dataset 自动生成评估基准。 + #### JSONL 文件格式示例 ```json diff --git a/server/routers/evaluation_router.py b/server/routers/evaluation_router.py index bdfcba5f..d67a8d4a 100644 --- a/server/routers/evaluation_router.py +++ b/server/routers/evaluation_router.py @@ -14,15 +14,27 @@ evaluation = APIRouter(prefix="/evaluation", tags=["evaluation"]) @evaluation.get("/databases/{db_id}/benchmarks/{benchmark_id}") async def get_evaluation_benchmark_by_db( - db_id: str, benchmark_id: str, current_user: User = Depends(get_admin_user) + db_id: str, + benchmark_id: str, + page: int = 1, + page_size: int = 10, + current_user: User = Depends(get_admin_user) ): - """根据 db_id 获取评估基准详情""" + """根据 db_id 获取评估基准详情（支持分页）""" from src.services.evaluation_service import EvaluationService try: + # 验证分页参数 + if page < 1: + raise HTTPException(status_code=400, detail="页码必须大于0") + if page_size < 1 or page_size > 100: + raise HTTPException(status_code=400, detail="每页大小必须在1-100之间") + service = EvaluationService() - benchmark = await service.get_benchmark_detail_by_db(db_id, benchmark_id) + benchmark = await service.get_benchmark_detail_by_db(db_id, benchmark_id, page, page_size) return {"message": "success", "data": benchmark} + except HTTPException: + raise except Exception as e: logger.error(f"获取评估基准详情失败: {e}, {traceback.format_exc()}") raise HTTPException(status_code=500, detail=f"获取评估基准详情失败: {str(e)}") @@ -44,13 +56,28 @@ async def delete_evaluation_benchmark(benchmark_id: str, current_user: User = De @evaluation.get("/databases/{db_id}/results/{task_id}") -async def get_evaluation_results_by_db(db_id: str, task_id: str, current_user: User = Depends(get_admin_user)): - """获取评估结果（带 db_id）""" +async def get_evaluation_results_by_db( + db_id: str, + task_id: str, + page: int = 1, + page_size: int = 20, + error_only: bool = False, + current_user: User = Depends(get_admin_user) +): + """获取评估结果（带 db_id，支持分页）""" from src.services.evaluation_service import EvaluationService try: + # 验证分页参数 + if page < 1: + raise HTTPException(status_code=400, detail="页码必须大于0") + if page_size < 1 or page_size > 100: + raise HTTPException(status_code=400, detail="每页大小必须在1-100之间") + service = EvaluationService() - results = await service.get_evaluation_results_by_db(db_id, task_id) + results = await service.get_evaluation_results_by_db( + db_id, task_id, page=page, page_size=page_size, error_only=error_only + ) return {"message": "success", "data": results} except Exception as e: logger.error(f"获取评估结果失败: {e}, {traceback.format_exc()}") diff --git a/src/services/evaluation_service.py b/src/services/evaluation_service.py index b3afcf17..96ab6dbf 100644 --- a/src/services/evaluation_service.py +++ b/src/services/evaluation_service.py @@ -136,8 +136,8 @@ class EvaluationService: logger.error(f"获取评估基准详情失败: {e}") raise - async def get_benchmark_detail_by_db(self, db_id: str, benchmark_id: str) -> dict[str, Any]: - """根据 db_id 直接获取评估基准详情""" + async def get_benchmark_detail_by_db(self, db_id: str, benchmark_id: str, page: int = 1, page_size: int = 10) -> dict[str, Any]: + """根据 db_id 获取评估基准详情（支持分页）""" try: kb_instance = knowledge_base.get_kb(db_id) benchmarks_map = kb_instance.benchmarks_meta.get(db_id, {}) @@ -145,14 +145,46 @@ class EvaluationService: raise ValueError("Benchmark not found") meta = benchmarks_map[benchmark_id] data_file_path = meta.get("benchmark_file") + + # 获取总问题数和分页数据 + total_questions = meta.get("question_count", 0) questions = [] + if data_file_path and os.path.exists(data_file_path): + # 计算分页范围 + start_index = (page - 1) * page_size + end_index = start_index + page_size + + # 读取指定范围的问题 with open(data_file_path, encoding="utf-8") as f: + current_index = 0 for line in f: - if line.strip(): + if not line.strip(): + continue + + # 只处理指定范围内的问题 + if current_index >= start_index and current_index < end_index: questions.append(json.loads(line)) + elif current_index >= end_index: + break # 已经读取到足够的问题，停止读取 + + current_index += 1 + + # 计算分页信息 + total_pages = (total_questions + page_size - 1) // page_size + meta_with_q = meta.copy() - meta_with_q["questions"] = questions + meta_with_q.update({ + "questions": questions, + "pagination": { + "current_page": page, + "page_size": page_size, + "total_questions": total_questions, + "total_pages": total_pages, + "has_next": page < total_pages, + "has_prev": page > 1 + } + }) return meta_with_q except Exception as e: logger.error(f"获取评估基准详情失败: {e}") @@ -556,7 +588,8 @@ class EvaluationService: f"基于以下上下文信息，请回答用户的问题。\n\n" f"上下文信息：{context_text}\n\n" f"用户问题：{question_data["query"]}\n\n" - "请根据上下文信息准确回答问题。如果上下文中没有相关信息，请说明。\n\n" + "请根据上下文信息准确回答问题。\n\n" + "如果上下文中缺少相关信息，请回答“信息不足，无法回答”。\n\n" ) # 生成答案 - 使用 asyncio.to_thread 避免阻塞事件循环 @@ -728,7 +761,14 @@ class EvaluationService: raise # 索引与回退逻辑已移除，统一通过 db_id 定位 - async def get_evaluation_results_by_db(self, db_id: str, task_id: str) -> dict[str, Any]: + async def get_evaluation_results_by_db( + self, + db_id: str, + task_id: str, + page: int = 1, + page_size: int = 20, + error_only: bool = False + ) -> dict[str, Any]: result_file_path = os.path.join(self._get_result_dir(db_id), f"{task_id}.json") if not os.path.exists(result_file_path): task = await tasker.get_task(task_id) @@ -740,8 +780,63 @@ class EvaluationService: "message": task.message, } raise ValueError(f"Result not found for task {task_id}") + + # 加载JSON文件 with open(result_file_path, encoding="utf-8") as f: - return json.load(f) + data = json.load(f) + + # 如果是分页请求，处理详细结果 + if page and page_size: + all_results = data.get("interim_results", data.get("results", [])) + + # 如果只要错误结果，先过滤 + if error_only: + filtered_results = [] + for item in all_results: + # 检查答案评分是否为错误（score <= 0.5） + if item.get("metrics", {}).get("score", 1.0) <= 0.5: + filtered_results.append(item) + continue + + # 检查检索指标是否明显偏低 + metrics = item.get("metrics", {}) + has_low_recall = any( + metrics.get(k, 1.0) < 0.3 + for k in metrics + if k.startswith("recall@") + ) + if has_low_recall: + filtered_results.append(item) + all_results = filtered_results + + # 计算分页 + total = len(all_results) + start_idx = (page - 1) * page_size + end_idx = start_idx + page_size + paged_results = all_results[start_idx:end_idx] + + # 返回分页数据 + return { + "task_id": data.get("task_id", task_id), + "status": data.get("status"), + "started_at": data.get("started_at"), + "completed_at": data.get("completed_at"), + "total_questions": data.get("total_questions", 0), + "completed_questions": data.get("completed_questions", 0), + "overall_score": data.get("overall_score"), + "retrieval_config": data.get("retrieval_config"), + "interim_results": paged_results, + "pagination": { + "current_page": page, + "page_size": page_size, + "total": total, + "total_pages": (total + page_size - 1) // page_size, + "error_only": error_only + } + } + + # 非分页请求，返回完整数据（保持向后兼容） + return data async def delete_evaluation_result_by_db(self, db_id: str, task_id: str) -> None: result_file_path = os.path.join(self._get_result_dir(db_id), f"{task_id}.json") diff --git a/web/src/apis/knowledge_api.js b/web/src/apis/knowledge_api.js index 272a33b9..63cb7a24 100644 --- a/web/src/apis/knowledge_api.js +++ b/web/src/apis/knowledge_api.js @@ -360,8 +360,12 @@ export const evaluationApi = { * @param {string} dbId - 知识库ID * @param {string} benchmarkId - 基准ID */ - getBenchmarkByDb: async (dbId, benchmarkId) => { - return apiAdminGet(`/api/evaluation/databases/${dbId}/benchmarks/${benchmarkId}`) + getBenchmarkByDb: async (dbId, benchmarkId, page = 1, pageSize = 50) => { + const params = new URLSearchParams({ + page: page.toString(), + page_size: pageSize.toString() + }) + return apiAdminGet(`/api/evaluation/databases/${dbId}/benchmarks/${benchmarkId}?${params}`) }, /** @@ -420,8 +424,15 @@ export const evaluationApi = { }, // 新接口：带 db_id 的评估结果查询与删除 - getEvaluationResultsByDb: async (dbId, taskId) => { - return apiAdminGet(`/api/evaluation/databases/${dbId}/results/${taskId}`) + getEvaluationResultsByDb: async (dbId, taskId, params = {}) => { + const queryParams = new URLSearchParams(); + + if (params.page) queryParams.append('page', params.page); + if (params.pageSize) queryParams.append('page_size', params.pageSize); + if (params.errorOnly !== undefined) queryParams.append('error_only', params.errorOnly); + + const url = `/api/evaluation/databases/${dbId}/results/${taskId}${queryParams.toString() ? '?' + queryParams.toString() : ''}`; + return apiAdminGet(url); }, deleteEvaluationResultByDb: async (dbId, taskId) => { return apiAdminDelete(`/api/evaluation/databases/${dbId}/results/${taskId}`) diff --git a/web/src/components/EvaluationBenchmarks.vue b/web/src/components/EvaluationBenchmarks.vue index 2d743442..4254a62c 100644 --- a/web/src/components/EvaluationBenchmarks.vue +++ b/web/src/components/EvaluationBenchmarks.vue @@ -120,7 +120,7 @@
@@ -146,30 +146,44 @@
-
-

问题示例 (前5条)

-
-
-
- Q{{ index + 1 }} -
-
-

{{ item.query }}

-
- 黄金Chunk: {{ item.gold_chunk_ids.slice(0, 3).join(', ') }} - ...等{{ item.gold_chunk_ids.length }}个 -
-
- 黄金答案: {{ item.gold_answer.slice(0, 150) }} - ... -
-
-
-
+
+

问题列表 (共{{ previewPagination.total }}条)

+ + +
@@ -177,7 +191,7 @@