651 lines
27 KiB
Python
651 lines
27 KiB
Python
import json
|
||
import os
|
||
import re
|
||
import uuid
|
||
from datetime import datetime
|
||
from typing import Any
|
||
|
||
from yuxi.knowledge import knowledge_base
|
||
from yuxi.knowledge.eval.benchmark_generation import dump_benchmark_item, iter_generated_benchmark_items
|
||
from yuxi.knowledge.eval.evaluator import aggregate_metrics, evaluate_question
|
||
from yuxi.models import select_model
|
||
from yuxi.repositories.evaluation_repository import EvaluationRepository
|
||
from yuxi.repositories.knowledge_base_repository import KnowledgeBaseRepository
|
||
from yuxi.services.task_service import TaskContext, tasker
|
||
from yuxi.utils import logger
|
||
|
||
|
||
class EvaluationService:
|
||
"""RAG评估服务"""
|
||
|
||
def __init__(self):
|
||
self.eval_repo = EvaluationRepository()
|
||
self.kb_repo = KnowledgeBaseRepository()
|
||
|
||
async def _get_benchmark_dir(self, db_id: str) -> str:
|
||
"""获取评估基准目录"""
|
||
kb_instance = await knowledge_base.aget_kb(db_id)
|
||
base_dir = os.path.join(kb_instance.work_dir, db_id)
|
||
path = os.path.join(base_dir, "benchmarks")
|
||
os.makedirs(path, exist_ok=True)
|
||
return path
|
||
|
||
async def _get_result_dir(self, db_id: str) -> str:
|
||
"""获取评估结果目录"""
|
||
kb_instance = await knowledge_base.aget_kb(db_id)
|
||
base_dir = os.path.join(kb_instance.work_dir, db_id)
|
||
path = os.path.join(base_dir, "results")
|
||
os.makedirs(path, exist_ok=True)
|
||
return path
|
||
|
||
# 已移除基准回退逻辑，统一使用集中元数据
|
||
|
||
# 已移除结果回退逻辑，统一通过 db_id 定位
|
||
|
||
async def upload_benchmark(
|
||
self, db_id: str, file_content: bytes, filename: str, name: str, description: str, created_by: str
|
||
) -> dict[str, Any]:
|
||
"""上传评估基准文件"""
|
||
try:
|
||
content_str = file_content.decode("utf-8")
|
||
questions = []
|
||
has_gold_chunks = False
|
||
has_gold_answers = False
|
||
|
||
# 解析 JSONL
|
||
for line_num, line in enumerate(content_str.strip().split("\n"), 1):
|
||
if not line.strip():
|
||
continue
|
||
try:
|
||
item = json.loads(line)
|
||
if "query" not in item:
|
||
raise ValueError(f"第{line_num}行缺少必需的'query'字段")
|
||
if item.get("gold_chunk_ids"):
|
||
has_gold_chunks = True
|
||
if item.get("gold_answer"):
|
||
has_gold_answers = True
|
||
questions.append(item)
|
||
except json.JSONDecodeError as e:
|
||
raise ValueError(f"第{line_num}行JSON格式错误: {str(e)}")
|
||
|
||
if not questions:
|
||
raise ValueError("文件中没有有效的问题数据")
|
||
|
||
benchmark_id = f"benchmark_{uuid.uuid4().hex[:8]}"
|
||
benchmark_dir = await self._get_benchmark_dir(db_id)
|
||
|
||
# 保存数据文件 (.jsonl)
|
||
data_file_path = os.path.join(benchmark_dir, f"{benchmark_id}.jsonl")
|
||
with open(data_file_path, "w", encoding="utf-8") as f:
|
||
f.write(content_str)
|
||
|
||
meta = {
|
||
"id": benchmark_id,
|
||
"benchmark_id": benchmark_id,
|
||
"name": name,
|
||
"description": description,
|
||
"db_id": db_id,
|
||
"question_count": len(questions),
|
||
"has_gold_chunks": has_gold_chunks,
|
||
"has_gold_answers": has_gold_answers,
|
||
"benchmark_file": data_file_path,
|
||
"created_by": created_by,
|
||
"created_at": datetime.utcnow().isoformat(),
|
||
"updated_at": datetime.utcnow().isoformat(),
|
||
}
|
||
await self.eval_repo.create_benchmark(
|
||
{
|
||
"benchmark_id": benchmark_id,
|
||
"db_id": db_id,
|
||
"name": name,
|
||
"description": description,
|
||
"question_count": len(questions),
|
||
"has_gold_chunks": has_gold_chunks,
|
||
"has_gold_answers": has_gold_answers,
|
||
"data_file_path": data_file_path,
|
||
"created_by": created_by,
|
||
}
|
||
)
|
||
return meta
|
||
|
||
except Exception as e:
|
||
logger.error(f"上传评估基准失败: {e}")
|
||
raise
|
||
|
||
async def get_benchmarks(self, db_id: str) -> list[dict[str, Any]]:
|
||
"""获取知识库的评估基准列表"""
|
||
try:
|
||
rows = await self.eval_repo.list_benchmarks(db_id)
|
||
return [
|
||
{
|
||
"id": row.benchmark_id,
|
||
"benchmark_id": row.benchmark_id,
|
||
"name": row.name,
|
||
"description": row.description,
|
||
"db_id": row.db_id,
|
||
"question_count": row.question_count,
|
||
"has_gold_chunks": row.has_gold_chunks,
|
||
"has_gold_answers": row.has_gold_answers,
|
||
"benchmark_file": row.data_file_path,
|
||
"created_by": row.created_by,
|
||
"created_at": row.created_at.isoformat() if row.created_at else None,
|
||
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
|
||
}
|
||
for row in rows
|
||
]
|
||
|
||
except Exception as e:
|
||
logger.error(f"获取评估基准列表失败: {e}")
|
||
raise
|
||
|
||
async def get_benchmark_detail(self, benchmark_id: str) -> dict[str, Any]:
|
||
"""获取评估基准详情 (包含问题列表)"""
|
||
try:
|
||
row = await self.eval_repo.get_benchmark(benchmark_id)
|
||
if row is None:
|
||
raise ValueError("Benchmark not found")
|
||
questions = []
|
||
if row.data_file_path and os.path.exists(row.data_file_path):
|
||
with open(row.data_file_path, encoding="utf-8") as f:
|
||
for line in f:
|
||
if line.strip():
|
||
questions.append(json.loads(line))
|
||
return {
|
||
"id": row.benchmark_id,
|
||
"benchmark_id": row.benchmark_id,
|
||
"name": row.name,
|
||
"description": row.description,
|
||
"db_id": row.db_id,
|
||
"question_count": row.question_count,
|
||
"has_gold_chunks": row.has_gold_chunks,
|
||
"has_gold_answers": row.has_gold_answers,
|
||
"benchmark_file": row.data_file_path,
|
||
"created_by": row.created_by,
|
||
"created_at": row.created_at.isoformat() if row.created_at else None,
|
||
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
|
||
"questions": questions,
|
||
}
|
||
|
||
except Exception as e:
|
||
logger.error(f"获取评估基准详情失败: {e}")
|
||
raise
|
||
|
||
async def get_benchmark_detail_by_db(
|
||
self, db_id: str, benchmark_id: str, page: int = 1, page_size: int = 10
|
||
) -> dict[str, Any]:
|
||
"""根据 db_id 获取评估基准详情（支持分页）"""
|
||
try:
|
||
row = await self.eval_repo.get_benchmark(benchmark_id)
|
||
if row is None or row.db_id != db_id:
|
||
raise ValueError("Benchmark not found")
|
||
data_file_path = row.data_file_path
|
||
total_questions = row.question_count or 0
|
||
questions = []
|
||
|
||
if data_file_path and os.path.exists(data_file_path):
|
||
# 计算分页范围
|
||
start_index = (page - 1) * page_size
|
||
end_index = start_index + page_size
|
||
|
||
# 读取指定范围的问题
|
||
with open(data_file_path, encoding="utf-8") as f:
|
||
current_index = 0
|
||
for line in f:
|
||
if not line.strip():
|
||
continue
|
||
|
||
# 只处理指定范围内的问题
|
||
if current_index >= start_index and current_index < end_index:
|
||
questions.append(json.loads(line))
|
||
elif current_index >= end_index:
|
||
break # 已经读取到足够的问题，停止读取
|
||
|
||
current_index += 1
|
||
|
||
# 计算分页信息
|
||
total_pages = (total_questions + page_size - 1) // page_size
|
||
|
||
return {
|
||
"id": row.benchmark_id,
|
||
"benchmark_id": row.benchmark_id,
|
||
"name": row.name,
|
||
"description": row.description,
|
||
"db_id": row.db_id,
|
||
"question_count": row.question_count,
|
||
"has_gold_chunks": row.has_gold_chunks,
|
||
"has_gold_answers": row.has_gold_answers,
|
||
"benchmark_file": data_file_path,
|
||
"created_by": row.created_by,
|
||
"created_at": row.created_at.isoformat() if row.created_at else None,
|
||
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
|
||
"questions": questions,
|
||
"pagination": {
|
||
"current_page": page,
|
||
"page_size": page_size,
|
||
"total_questions": total_questions,
|
||
"total_pages": total_pages,
|
||
"has_next": page < total_pages,
|
||
"has_prev": page > 1,
|
||
},
|
||
}
|
||
except Exception as e:
|
||
logger.error(f"获取评估基准详情失败: {e}")
|
||
raise
|
||
|
||
async def get_benchmark_download_info(self, benchmark_id: str) -> dict[str, str]:
|
||
"""获取评估基准下载信息"""
|
||
row = await self.eval_repo.get_benchmark(benchmark_id)
|
||
if row is None:
|
||
raise ValueError("Benchmark not found")
|
||
|
||
data_file_path = row.data_file_path or ""
|
||
if not data_file_path or not os.path.exists(data_file_path):
|
||
raise ValueError("Benchmark file not found")
|
||
|
||
filename_base = (row.name or "").strip()
|
||
if not filename_base:
|
||
filename_base = row.benchmark_id
|
||
|
||
filename_base = re.sub(r"[\\/:*?\"<>|]+", "_", filename_base).strip()
|
||
if not filename_base or filename_base in {".", ".."}:
|
||
filename_base = row.benchmark_id
|
||
|
||
if not filename_base.endswith(".jsonl"):
|
||
filename_base = f"{filename_base}.jsonl"
|
||
|
||
return {"file_path": data_file_path, "filename": filename_base}
|
||
|
||
async def delete_benchmark(self, benchmark_id: str) -> None:
|
||
"""删除评估基准"""
|
||
try:
|
||
row = await self.eval_repo.get_benchmark(benchmark_id)
|
||
if row is None:
|
||
raise ValueError("Benchmark not found")
|
||
if row.data_file_path and os.path.exists(row.data_file_path):
|
||
os.remove(row.data_file_path)
|
||
await self.eval_repo.delete_benchmark(benchmark_id)
|
||
logger.info(f"成功删除评估基准: {benchmark_id}")
|
||
return
|
||
|
||
except Exception as e:
|
||
logger.error(f"删除评估基准失败: {e}")
|
||
raise
|
||
|
||
async def delete_evaluation_result(self, task_id: str, db_id: str) -> None:
|
||
"""删除评估结果"""
|
||
if not task_id:
|
||
raise ValueError("task_id is required")
|
||
await self.delete_evaluation_result_by_db(db_id, task_id)
|
||
|
||
async def generate_benchmark(self, db_id: str, params: dict[str, Any], created_by: str) -> dict[str, Any]:
|
||
task_id = f"gen_benchmark_{uuid.uuid4().hex[:8]}"
|
||
await tasker.enqueue(
|
||
name="生成评估基准",
|
||
task_type="benchmark_generation",
|
||
payload={"task_id": task_id, "db_id": db_id, "created_by": created_by, **params},
|
||
coroutine=self._generate_benchmark_task,
|
||
)
|
||
return {"task_id": task_id, "message": "基准生成任务已提交"}
|
||
|
||
async def _generate_benchmark_task(self, context: TaskContext):
|
||
await context.set_progress(0, "初始化")
|
||
|
||
task = context._tasker._tasks.get(context.task_id)
|
||
payload = task.payload if task else {}
|
||
|
||
db_id = payload.get("db_id")
|
||
name = payload.get("name", "自动生成评估基准")
|
||
description = payload.get("description", "")
|
||
count = int(payload.get("count", 10))
|
||
neighbors_count = int(payload.get("neighbors_count", 1))
|
||
llm_model_spec = payload.get("llm_model_spec")
|
||
|
||
kb_instance = await knowledge_base.aget_kb(db_id)
|
||
if not kb_instance:
|
||
await context.set_message("知识库不存在")
|
||
raise ValueError("Knowledge Base not found")
|
||
if kb_instance.kb_type != "milvus":
|
||
await context.set_message("仅支持 commonrag/Milvus 类型知识库生成评估基准")
|
||
raise ValueError("Unsupported KB type for benchmark generation")
|
||
|
||
benchmark_id = f"benchmark_{uuid.uuid4().hex[:8]}"
|
||
bench_dir = await self._get_benchmark_dir(db_id)
|
||
data_file_path = os.path.join(bench_dir, f"{benchmark_id}.jsonl")
|
||
generated = 0
|
||
|
||
try:
|
||
with open(data_file_path, "w", encoding="utf-8") as f:
|
||
async for item in iter_generated_benchmark_items(
|
||
kb_instance=kb_instance,
|
||
db_id=db_id,
|
||
count=count,
|
||
neighbors_count=neighbors_count,
|
||
llm_model_spec=llm_model_spec,
|
||
progress_cb=context.set_progress,
|
||
):
|
||
f.write(dump_benchmark_item(item))
|
||
generated += 1
|
||
except ValueError as e:
|
||
if str(e) == "No chunks found in knowledge base":
|
||
await context.set_message("知识库为空或未解析到chunks")
|
||
raise
|
||
|
||
await self.eval_repo.create_benchmark(
|
||
{
|
||
"benchmark_id": benchmark_id,
|
||
"db_id": db_id,
|
||
"name": name,
|
||
"description": description,
|
||
"question_count": generated,
|
||
"has_gold_chunks": True,
|
||
"has_gold_answers": True,
|
||
"data_file_path": data_file_path,
|
||
"created_by": payload.get("created_by"),
|
||
}
|
||
)
|
||
|
||
await context.set_progress(100, "完成")
|
||
|
||
async def run_evaluation(
|
||
self, db_id: str, benchmark_id: str, model_config: dict[str, Any] = None, created_by: str = "system"
|
||
) -> str:
|
||
"""运行RAG评估"""
|
||
try:
|
||
task_id = f"eval_{uuid.uuid4().hex[:8]}"
|
||
|
||
benchmark_row = await self.eval_repo.get_benchmark(benchmark_id)
|
||
if benchmark_row is None or benchmark_row.db_id != db_id:
|
||
raise ValueError("Benchmark not found")
|
||
|
||
# 从知识库元数据中获取检索配置
|
||
retrieval_config = {}
|
||
try:
|
||
kb_row = await self.kb_repo.get_by_id(db_id)
|
||
query_params = (kb_row.query_params if kb_row else None) or {}
|
||
retrieval_config = query_params.get("options", {}) if isinstance(query_params, dict) else {}
|
||
if not retrieval_config:
|
||
kb_instance = await knowledge_base.aget_kb(db_id)
|
||
if kb_instance:
|
||
retrieval_config = kb_instance._get_default_query_params(db_id).get("options", {})
|
||
logger.info(f"从知识库 {db_id} 加载检索配置: {list(retrieval_config.keys())}")
|
||
except Exception as e:
|
||
logger.error(f"获取知识库检索配置失败: {e}")
|
||
# 使用空配置作为默认值
|
||
|
||
# 合并前端传递的模型配置
|
||
if model_config:
|
||
retrieval_config.update(model_config)
|
||
|
||
await self.eval_repo.create_result(
|
||
{
|
||
"task_id": task_id,
|
||
"db_id": db_id,
|
||
"benchmark_id": benchmark_id,
|
||
"status": "running",
|
||
"retrieval_config": retrieval_config,
|
||
"metrics": {},
|
||
"overall_score": None,
|
||
"total_questions": benchmark_row.question_count or 0,
|
||
"completed_questions": 0,
|
||
"started_at": datetime.utcnow(),
|
||
"completed_at": None,
|
||
"created_by": created_by,
|
||
}
|
||
)
|
||
|
||
await tasker.enqueue(
|
||
name=f"RAG评估({benchmark_row.name})",
|
||
task_type="rag_evaluation",
|
||
payload={
|
||
"task_id": task_id,
|
||
"db_id": db_id,
|
||
"benchmark_id": benchmark_id,
|
||
"retrieval_config": retrieval_config,
|
||
"created_by": created_by,
|
||
},
|
||
coroutine=self._run_evaluation_task,
|
||
)
|
||
|
||
return task_id
|
||
|
||
except Exception as e:
|
||
logger.error(f"启动评估失败: {e}")
|
||
raise
|
||
|
||
async def _run_evaluation_task(self, context: TaskContext):
|
||
"""运行评估任务"""
|
||
try:
|
||
task = context._tasker._tasks.get(context.task_id)
|
||
if not task:
|
||
raise ValueError("Task not found")
|
||
payload = task.payload
|
||
|
||
task_id = payload["task_id"]
|
||
db_id = payload["db_id"]
|
||
benchmark_id = payload["benchmark_id"]
|
||
retrieval_config = payload["retrieval_config"]
|
||
|
||
# 加载基准数据
|
||
await context.set_progress(5, "加载基准数据")
|
||
benchmark_row = await self.eval_repo.get_benchmark(benchmark_id)
|
||
if benchmark_row is None or benchmark_row.db_id != db_id:
|
||
raise ValueError("Benchmark not found")
|
||
data_path = benchmark_row.data_file_path
|
||
if not data_path or not os.path.exists(data_path):
|
||
raise ValueError("Benchmark file not found")
|
||
|
||
benchmark_data = []
|
||
with open(data_path, encoding="utf-8") as f:
|
||
for line in f:
|
||
if line.strip():
|
||
benchmark_data.append(json.loads(line))
|
||
|
||
# 开始评估
|
||
kb_instance = await knowledge_base.aget_kb(db_id)
|
||
if not kb_instance:
|
||
raise ValueError(f"Knowledge Base {db_id} not found")
|
||
|
||
if kb_instance.kb_type == "lightrag":
|
||
raise ValueError("暂不支持对 LightRAG 类型的知识库进行 RAG 评估")
|
||
|
||
# 初始化 Judge LLM
|
||
judge_llm = None
|
||
if benchmark_row.has_gold_answers:
|
||
# 优先使用配置中的 judge_llm，否则回退到 answer_llm，或者默认
|
||
judge_model_spec = retrieval_config.get("judge_llm") or retrieval_config.get("answer_llm")
|
||
if judge_model_spec:
|
||
try:
|
||
logger.debug(f"Initializing Judge LLM: {judge_model_spec}")
|
||
judge_llm = select_model(model_spec=judge_model_spec)
|
||
except Exception as e:
|
||
logger.error(f"Failed to load judge LLM: {e}")
|
||
|
||
total_questions = len(benchmark_data)
|
||
all_retrieval_metrics = []
|
||
all_answer_metrics = []
|
||
|
||
async def update_result_db(
|
||
status: str | None = None, completed: int | None = None, metrics=None, final_score=None
|
||
):
|
||
payload = {}
|
||
if status is not None:
|
||
payload["status"] = status
|
||
if status in ["completed", "failed"]:
|
||
payload["completed_at"] = datetime.utcnow()
|
||
if completed is not None:
|
||
payload["completed_questions"] = completed
|
||
if metrics is not None:
|
||
payload["metrics"] = metrics
|
||
if final_score is not None:
|
||
payload["overall_score"] = final_score
|
||
if payload:
|
||
await self.eval_repo.update_result(task_id, payload)
|
||
|
||
for i, question_data in enumerate(benchmark_data):
|
||
await context.raise_if_cancelled()
|
||
progress = 10 + (i / total_questions) * 80
|
||
await context.set_progress(progress, f"评估 {i + 1}/{total_questions}")
|
||
|
||
question_result = await evaluate_question(
|
||
kb_instance=kb_instance,
|
||
db_id=db_id,
|
||
question_data=question_data,
|
||
retrieval_config=retrieval_config,
|
||
has_gold_chunks=benchmark_row.has_gold_chunks,
|
||
has_gold_answers=benchmark_row.has_gold_answers,
|
||
judge_llm=judge_llm,
|
||
select_model_fn=select_model,
|
||
)
|
||
|
||
if benchmark_row.has_gold_chunks and question_data.get("gold_chunk_ids"):
|
||
all_retrieval_metrics.append(question_result["retrieval_scores"])
|
||
if benchmark_row.has_gold_answers and question_data.get("gold_answer") and judge_llm:
|
||
all_answer_metrics.append(question_result["answer_scores"])
|
||
|
||
await self.eval_repo.upsert_result_detail(
|
||
task_id=task_id,
|
||
query_index=i,
|
||
data=question_result["detail"],
|
||
)
|
||
|
||
current_overall_metrics, _ = aggregate_metrics(all_retrieval_metrics, all_answer_metrics)
|
||
await context.set_result(
|
||
{
|
||
"current_metrics": current_overall_metrics,
|
||
"completed_questions": i + 1,
|
||
"total_questions": total_questions,
|
||
}
|
||
)
|
||
|
||
if (i + 1) % 5 == 0 or (i + 1) == total_questions:
|
||
await update_result_db(completed=i + 1)
|
||
|
||
await context.set_progress(95, "计算最终指标")
|
||
overall_metrics, overall_score = aggregate_metrics(
|
||
all_retrieval_metrics, all_answer_metrics, include_overall_score=True
|
||
)
|
||
|
||
await update_result_db(
|
||
status="completed",
|
||
completed=total_questions,
|
||
metrics=overall_metrics,
|
||
final_score=overall_score,
|
||
)
|
||
await context.set_progress(100, "完成")
|
||
|
||
except Exception as e:
|
||
logger.error(f"Task failed: {e}")
|
||
try:
|
||
if "payload" in locals():
|
||
await self.eval_repo.update_result(
|
||
payload["task_id"],
|
||
{"status": "failed", "metrics": {"error": str(e)}, "completed_at": datetime.utcnow()},
|
||
)
|
||
except Exception as exc:
|
||
logger.error(f"Error updating result record: {exc}")
|
||
|
||
await context.set_message(f"Error: {str(e)}")
|
||
raise
|
||
|
||
async def get_evaluation_results(self, task_id: str, db_id: str) -> dict[str, Any]:
|
||
"""获取评估结果"""
|
||
if not task_id:
|
||
raise ValueError("task_id is required")
|
||
return await self.get_evaluation_results_by_db(db_id, task_id)
|
||
|
||
async def get_evaluation_history(self, db_id: str) -> list[dict[str, Any]]:
|
||
"""获取知识库的评估历史记录"""
|
||
try:
|
||
rows = await self.eval_repo.list_results(db_id)
|
||
return [
|
||
{
|
||
"task_id": row.task_id,
|
||
"benchmark_id": row.benchmark_id,
|
||
"status": row.status,
|
||
"started_at": row.started_at.isoformat() if row.started_at else None,
|
||
"completed_at": row.completed_at.isoformat() if row.completed_at else None,
|
||
"total_questions": row.total_questions,
|
||
"completed_questions": row.completed_questions,
|
||
"overall_score": row.overall_score,
|
||
"retrieval_config": row.retrieval_config or {},
|
||
"metrics": row.metrics or {},
|
||
}
|
||
for row in rows
|
||
]
|
||
|
||
except Exception as e:
|
||
logger.error(f"获取评估历史失败: {e}")
|
||
raise
|
||
# 索引与回退逻辑已移除，统一通过 db_id 定位
|
||
|
||
async def get_evaluation_results_by_db(
|
||
self, db_id: str, task_id: str, page: int = 1, page_size: int = 20, error_only: bool = False
|
||
) -> dict[str, Any]:
|
||
if not re.match(r"^eval_[a-f0-9]{8}$", task_id):
|
||
raise ValueError("Invalid task_id format")
|
||
row = await self.eval_repo.get_result(task_id)
|
||
if row is None or row.db_id != db_id:
|
||
task = await tasker.get_task(task_id)
|
||
if task:
|
||
return {"task_id": task_id, "status": task.status, "progress": task.progress, "message": task.message}
|
||
raise ValueError(f"Result not found for task {task_id}")
|
||
|
||
details = await self.eval_repo.list_result_details(task_id)
|
||
all_results = [
|
||
{
|
||
"query": d.query_text,
|
||
"gold_chunk_ids": d.gold_chunk_ids,
|
||
"gold_answer": d.gold_answer,
|
||
"generated_answer": d.generated_answer,
|
||
"retrieved_chunks": d.retrieved_chunks,
|
||
"metrics": d.metrics or {},
|
||
}
|
||
for d in details
|
||
]
|
||
|
||
if error_only:
|
||
filtered_results = []
|
||
for item in all_results:
|
||
if item.get("metrics", {}).get("score", 1.0) <= 0.5:
|
||
filtered_results.append(item)
|
||
continue
|
||
metrics = item.get("metrics", {})
|
||
has_low_recall = any(metrics.get(k, 1.0) < 0.3 for k in metrics if k.startswith("recall@"))
|
||
if has_low_recall:
|
||
filtered_results.append(item)
|
||
all_results = filtered_results
|
||
|
||
total = len(all_results)
|
||
start_idx = (page - 1) * page_size
|
||
end_idx = start_idx + page_size
|
||
paged_results = all_results[start_idx:end_idx]
|
||
|
||
return {
|
||
"task_id": row.task_id,
|
||
"status": row.status,
|
||
"started_at": row.started_at.isoformat() if row.started_at else None,
|
||
"completed_at": row.completed_at.isoformat() if row.completed_at else None,
|
||
"total_questions": row.total_questions or 0,
|
||
"completed_questions": row.completed_questions or 0,
|
||
"overall_score": row.overall_score,
|
||
"retrieval_config": row.retrieval_config or {},
|
||
"interim_results": paged_results,
|
||
"pagination": {
|
||
"current_page": page,
|
||
"page_size": page_size,
|
||
"total": total,
|
||
"total_pages": (total + page_size - 1) // page_size,
|
||
"error_only": error_only,
|
||
},
|
||
}
|
||
|
||
async def delete_evaluation_result_by_db(self, db_id: str, task_id: str) -> None:
|
||
if not re.match(r"^eval_[a-f0-9]{8}$", task_id):
|
||
raise ValueError("Invalid task_id format")
|
||
row = await self.eval_repo.get_result(task_id)
|
||
if row is None or row.db_id != db_id:
|
||
raise ValueError("Result not found")
|
||
await self.eval_repo.delete_result(task_id)
|
||
logger.info(f"成功删除评估结果: {task_id}")
|
||
return
|