1efbf86bf0
- 评估数据集/题目/运行/逐题结果全量入库，JSONL 仅作为交换格式 - benchmark -> dataset, task -> run 术语统一 - 评估生成支持配置并发数 - 移除 base.py 中旧的 benchmarks_meta 元数据逻辑 - manager.py 统一表创建方法名
262 lines
10 KiB
Python
262 lines
10 KiB
Python
import traceback
|
|
from typing import Any
|
|
from urllib.parse import quote
|
|
|
|
from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile
|
|
from fastapi.responses import Response
|
|
from pydantic import BaseModel, Field
|
|
from server.utils.auth_middleware import get_admin_user
|
|
from yuxi.knowledge.eval.benchmark_generation import (
|
|
DEFAULT_BENCHMARK_GENERATION_CONCURRENCY,
|
|
MAX_BENCHMARK_GENERATION_CONCURRENCY,
|
|
)
|
|
from yuxi.storage.postgres.models_business import User
|
|
from yuxi.utils import logger
|
|
|
|
|
|
evaluation = APIRouter(prefix="/evaluation", tags=["evaluation"])
|
|
|
|
|
|
class GenerateDatasetRequest(BaseModel):
|
|
name: str = Field(default="自动生成评估数据集", min_length=1, max_length=100)
|
|
description: str = ""
|
|
count: int = Field(default=10, ge=1, le=100)
|
|
neighbors_count: int = Field(default=1, ge=0, le=10)
|
|
concurrency_count: int = Field(
|
|
default=DEFAULT_BENCHMARK_GENERATION_CONCURRENCY,
|
|
ge=1,
|
|
le=MAX_BENCHMARK_GENERATION_CONCURRENCY,
|
|
)
|
|
llm_model_spec: str = Field(..., min_length=1)
|
|
|
|
|
|
class RunEvaluationRequest(BaseModel):
|
|
dataset_id: str = Field(..., min_length=1)
|
|
retrieval_config: dict[str, Any] = Field(default_factory=dict, alias="model_config")
|
|
|
|
|
|
@evaluation.post("/databases/{db_id}/datasets/upload")
|
|
async def upload_evaluation_dataset(
|
|
db_id: str,
|
|
file: UploadFile = File(...),
|
|
name: str = Form(...),
|
|
description: str = Form(""),
|
|
current_user: User = Depends(get_admin_user),
|
|
):
|
|
"""上传评估数据集"""
|
|
from yuxi.services.evaluation_service import EvaluationService
|
|
|
|
try:
|
|
if not file.filename.endswith(".jsonl"):
|
|
raise HTTPException(status_code=400, detail="仅支持JSONL格式文件")
|
|
|
|
service = EvaluationService()
|
|
result = await service.upload_dataset(
|
|
db_id=db_id,
|
|
file_content=await file.read(),
|
|
filename=file.filename,
|
|
name=name,
|
|
description=description,
|
|
created_by=current_user.uid,
|
|
)
|
|
return {"message": "success", "data": result}
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"上传评估数据集失败: {e}, {traceback.format_exc()}")
|
|
raise HTTPException(status_code=500, detail=f"上传评估数据集失败: {str(e)}")
|
|
|
|
|
|
@evaluation.get("/databases/{db_id}/datasets")
|
|
async def list_evaluation_datasets(db_id: str, current_user: User = Depends(get_admin_user)):
|
|
"""获取知识库的评估数据集列表"""
|
|
from yuxi.services.evaluation_service import EvaluationService
|
|
|
|
try:
|
|
service = EvaluationService()
|
|
datasets = await service.list_datasets(db_id)
|
|
return {"message": "success", "data": datasets}
|
|
except Exception as e:
|
|
logger.error(f"获取评估数据集列表失败: {e}, {traceback.format_exc()}")
|
|
raise HTTPException(status_code=500, detail=f"获取评估数据集列表失败: {str(e)}")
|
|
|
|
|
|
@evaluation.get("/databases/{db_id}/datasets/{dataset_id}")
|
|
async def get_evaluation_dataset(
|
|
db_id: str, dataset_id: str, page: int = 1, page_size: int = 10, current_user: User = Depends(get_admin_user)
|
|
):
|
|
"""获取评估数据集详情"""
|
|
from yuxi.services.evaluation_service import EvaluationService
|
|
|
|
try:
|
|
if page < 1:
|
|
raise HTTPException(status_code=400, detail="页码必须大于0")
|
|
if page_size < 1 or page_size > 100:
|
|
raise HTTPException(status_code=400, detail="每页大小必须在1-100之间")
|
|
|
|
service = EvaluationService()
|
|
dataset = await service.get_dataset_detail(db_id, dataset_id, page, page_size)
|
|
return {"message": "success", "data": dataset}
|
|
except HTTPException:
|
|
raise
|
|
except ValueError as e:
|
|
if "not found" in str(e).lower():
|
|
raise HTTPException(status_code=404, detail=str(e))
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
except Exception as e:
|
|
logger.error(f"获取评估数据集详情失败: {e}, {traceback.format_exc()}")
|
|
raise HTTPException(status_code=500, detail=f"获取评估数据集详情失败: {str(e)}")
|
|
|
|
|
|
@evaluation.get("/datasets/{dataset_id}/download")
|
|
async def download_evaluation_dataset(dataset_id: str, current_user: User = Depends(get_admin_user)):
|
|
"""导出评估数据集 JSONL"""
|
|
from yuxi.services.evaluation_service import EvaluationService
|
|
|
|
try:
|
|
service = EvaluationService()
|
|
export_info = await service.export_dataset_jsonl(dataset_id)
|
|
filename = export_info["filename"]
|
|
return Response(
|
|
content=export_info["content"].encode("utf-8"),
|
|
media_type="application/x-ndjson",
|
|
headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename)}"},
|
|
)
|
|
except ValueError as e:
|
|
if "not found" in str(e).lower():
|
|
raise HTTPException(status_code=404, detail=str(e))
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
except Exception as e:
|
|
logger.error(f"导出评估数据集失败: {e}, {traceback.format_exc()}")
|
|
raise HTTPException(status_code=500, detail=f"导出评估数据集失败: {str(e)}")
|
|
|
|
|
|
@evaluation.delete("/datasets/{dataset_id}")
|
|
async def delete_evaluation_dataset(dataset_id: str, current_user: User = Depends(get_admin_user)):
|
|
"""删除评估数据集"""
|
|
from yuxi.services.evaluation_service import EvaluationService
|
|
|
|
try:
|
|
service = EvaluationService()
|
|
await service.delete_dataset(dataset_id)
|
|
return {"message": "success", "data": None}
|
|
except ValueError as e:
|
|
if "not found" in str(e).lower():
|
|
raise HTTPException(status_code=404, detail=str(e))
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
except Exception as e:
|
|
logger.error(f"删除评估数据集失败: {e}, {traceback.format_exc()}")
|
|
raise HTTPException(status_code=500, detail=f"删除评估数据集失败: {str(e)}")
|
|
|
|
|
|
@evaluation.post("/databases/{db_id}/datasets/generate")
|
|
async def generate_evaluation_dataset(
|
|
db_id: str, request: GenerateDatasetRequest, current_user: User = Depends(get_admin_user)
|
|
):
|
|
"""自动生成评估数据集"""
|
|
from yuxi.services.evaluation_service import EvaluationService
|
|
|
|
try:
|
|
service = EvaluationService()
|
|
result = await service.generate_dataset(
|
|
db_id=db_id,
|
|
name=request.name,
|
|
description=request.description,
|
|
count=request.count,
|
|
neighbors_count=request.neighbors_count,
|
|
concurrency_count=request.concurrency_count,
|
|
llm_model_spec=request.llm_model_spec,
|
|
created_by=current_user.uid,
|
|
)
|
|
return {"message": "success", "data": result}
|
|
except Exception as e:
|
|
logger.error(f"生成评估数据集失败: {e}, {traceback.format_exc()}")
|
|
raise HTTPException(status_code=500, detail=f"生成评估数据集失败: {str(e)}")
|
|
|
|
|
|
@evaluation.post("/databases/{db_id}/runs")
|
|
async def run_evaluation(db_id: str, request: RunEvaluationRequest, current_user: User = Depends(get_admin_user)):
|
|
"""运行RAG评估"""
|
|
from yuxi.services.evaluation_service import EvaluationService
|
|
|
|
try:
|
|
service = EvaluationService()
|
|
run_id = await service.run_evaluation(
|
|
db_id=db_id,
|
|
dataset_id=request.dataset_id,
|
|
model_config=request.retrieval_config,
|
|
created_by=current_user.uid,
|
|
)
|
|
return {"message": "success", "data": {"run_id": run_id}}
|
|
except ValueError as e:
|
|
if "not found" in str(e).lower():
|
|
raise HTTPException(status_code=404, detail=str(e))
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
except Exception as e:
|
|
logger.error(f"启动评估失败: {e}, {traceback.format_exc()}")
|
|
raise HTTPException(status_code=500, detail=f"启动评估失败: {str(e)}")
|
|
|
|
|
|
@evaluation.get("/databases/{db_id}/runs")
|
|
async def list_evaluation_runs(db_id: str, current_user: User = Depends(get_admin_user)):
|
|
"""获取知识库评估运行历史"""
|
|
from yuxi.services.evaluation_service import EvaluationService
|
|
|
|
try:
|
|
service = EvaluationService()
|
|
runs = await service.list_runs(db_id)
|
|
return {"message": "success", "data": runs}
|
|
except Exception as e:
|
|
logger.error(f"获取评估运行历史失败: {e}, {traceback.format_exc()}")
|
|
raise HTTPException(status_code=500, detail=f"获取评估运行历史失败: {str(e)}")
|
|
|
|
|
|
@evaluation.get("/databases/{db_id}/runs/{run_id}")
|
|
async def get_evaluation_run_results(
|
|
db_id: str,
|
|
run_id: str,
|
|
page: int = 1,
|
|
page_size: int = 20,
|
|
error_only: bool = False,
|
|
current_user: User = Depends(get_admin_user),
|
|
):
|
|
"""获取评估运行结果"""
|
|
from yuxi.services.evaluation_service import EvaluationService
|
|
|
|
try:
|
|
if page < 1:
|
|
raise HTTPException(status_code=400, detail="页码必须大于0")
|
|
if page_size < 1 or page_size > 100:
|
|
raise HTTPException(status_code=400, detail="每页大小必须在1-100之间")
|
|
|
|
service = EvaluationService()
|
|
results = await service.get_run_results(db_id, run_id, page=page, page_size=page_size, error_only=error_only)
|
|
return {"message": "success", "data": results}
|
|
except HTTPException:
|
|
raise
|
|
except ValueError as e:
|
|
if "not found" in str(e).lower():
|
|
raise HTTPException(status_code=404, detail=str(e))
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
except Exception as e:
|
|
logger.error(f"获取评估运行结果失败: {e}, {traceback.format_exc()}")
|
|
raise HTTPException(status_code=500, detail=f"获取评估运行结果失败: {str(e)}")
|
|
|
|
|
|
@evaluation.delete("/databases/{db_id}/runs/{run_id}")
|
|
async def delete_evaluation_run(db_id: str, run_id: str, current_user: User = Depends(get_admin_user)):
|
|
"""删除评估运行"""
|
|
from yuxi.services.evaluation_service import EvaluationService
|
|
|
|
try:
|
|
service = EvaluationService()
|
|
await service.delete_run(db_id, run_id)
|
|
return {"message": "success", "data": None}
|
|
except ValueError as e:
|
|
if "not found" in str(e).lower():
|
|
raise HTTPException(status_code=404, detail=str(e))
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
except Exception as e:
|
|
logger.error(f"删除评估运行失败: {e}, {traceback.format_exc()}")
|
|
raise HTTPException(status_code=500, detail=f"删除评估运行失败: {str(e)}")
|