feat: 支持评估运行命名
This commit is contained in:
Wenjie Zhang
parent
f25a735566
commit
70b76f1e6d
@ -118,7 +118,7 @@ def aggregate_metrics(
|
||||
answer_metrics_list: list[dict[str, Any]],
|
||||
*,
|
||||
include_overall_score: bool = False,
|
||||
) -> tuple[dict[str, Any], float]:
|
||||
) -> tuple[dict[str, Any], float | None]:
|
||||
overall_metrics = {}
|
||||
|
||||
if retrieval_metrics_list:
|
||||
|
||||
@ -131,24 +131,13 @@ class EvaluationMetricsCalculator:
|
||||
@staticmethod
|
||||
def calculate_overall_score(
|
||||
retrieval_metrics_list: list[dict[str, float]], answer_metrics_list: list[dict[str, Any]]
|
||||
) -> float:
|
||||
"""计算整体平均分"""
|
||||
total_score = 0.0
|
||||
count = 0
|
||||
|
||||
# 简单的平均策略：将所有retrieval metric的值和answer metric的score一起平均
|
||||
# 用户可能希望分开看，但calculate_overall_score返回一个单值。
|
||||
) -> float | None:
|
||||
"""计算整体平均分，只聚合检索指标。"""
|
||||
retrieval_scores = []
|
||||
|
||||
# 计算检索平均分
|
||||
for m in retrieval_metrics_list:
|
||||
if m:
|
||||
total_score += sum(m.values()) / len(m)
|
||||
count += 1
|
||||
retrieval_scores.append(sum(m.values()) / len(m))
|
||||
|
||||
# 计算答案平均分
|
||||
for m in answer_metrics_list:
|
||||
if "score" in m:
|
||||
total_score += m["score"]
|
||||
count += 1
|
||||
|
||||
return total_score / count if count > 0 else 0.0
|
||||
return sum(retrieval_scores) / len(retrieval_scores) if retrieval_scores else None
|
||||
|
||||
@ -20,6 +20,14 @@ from yuxi.utils import logger
|
||||
from yuxi.utils.datetime_utils import format_utc_datetime, utc_now_naive
|
||||
|
||||
|
||||
def build_evaluation_run_name(started_at=None, hash_value: str | None = None) -> str:
|
||||
date_part = (started_at or utc_now_naive()).strftime("%Y%m%d")
|
||||
hash_part = re.sub(r"[^a-fA-F0-9]", "", hash_value or uuid.uuid4().hex).lower()[:6]
|
||||
if len(hash_part) < 6:
|
||||
hash_part = (hash_part + uuid.uuid4().hex)[:6]
|
||||
return f"eval-{date_part}-{hash_part}"
|
||||
|
||||
|
||||
class EvaluationService:
|
||||
"""RAG评估服务"""
|
||||
|
||||
@ -70,6 +78,18 @@ class EvaluationService:
|
||||
metrics.get(key, 1.0) < 0.3 for key in metrics if key.startswith("recall@")
|
||||
)
|
||||
|
||||
def _normalize_run_name(self, name: str | None, run_id: str) -> str:
|
||||
run_name = (name or "").strip()
|
||||
if run_name:
|
||||
return run_name
|
||||
return build_evaluation_run_name(hash_value=run_id.removeprefix("run_"))
|
||||
|
||||
def _run_name_from_row(self, row) -> str:
|
||||
name = (getattr(row, "name", None) or "").strip()
|
||||
if name:
|
||||
return name
|
||||
return build_evaluation_run_name(row.started_at, hash_value=row.run_id.removeprefix("run_"))
|
||||
|
||||
async def _sync_dataset_build_metadata(self, row) -> None:
|
||||
metadata = dict(row.build_metadata or {})
|
||||
if metadata.get("source") != "generated" or metadata.get("status") not in {"pending", "running"}:
|
||||
@ -419,10 +439,16 @@ class EvaluationService:
|
||||
raise
|
||||
|
||||
async def run_evaluation(
|
||||
self, kb_id: str, dataset_id: str, model_config: dict[str, Any] = None, created_by: str = "system"
|
||||
self,
|
||||
kb_id: str,
|
||||
dataset_id: str,
|
||||
name: str | None = None,
|
||||
model_config: dict[str, Any] = None,
|
||||
created_by: str = "system",
|
||||
) -> str:
|
||||
try:
|
||||
run_id = f"run_{uuid.uuid4().hex[:8]}"
|
||||
run_name = self._normalize_run_name(name, run_id)
|
||||
dataset_row = await self.eval_repo.get_dataset(dataset_id)
|
||||
if dataset_row is None or dataset_row.kb_id != kb_id:
|
||||
raise ValueError("Dataset not found")
|
||||
@ -448,6 +474,7 @@ class EvaluationService:
|
||||
await self.eval_repo.create_run(
|
||||
{
|
||||
"run_id": run_id,
|
||||
"name": run_name,
|
||||
"kb_id": kb_id,
|
||||
"dataset_id": dataset_id,
|
||||
"status": "running",
|
||||
@ -463,10 +490,11 @@ class EvaluationService:
|
||||
)
|
||||
|
||||
await tasker.enqueue(
|
||||
name=f"RAG评估({dataset_row.name})",
|
||||
name=f"RAG评估({run_name})",
|
||||
task_type="rag_evaluation",
|
||||
payload={
|
||||
"run_id": run_id,
|
||||
"name": run_name,
|
||||
"kb_id": kb_id,
|
||||
"dataset_id": dataset_id,
|
||||
"retrieval_config": retrieval_config,
|
||||
@ -614,6 +642,7 @@ class EvaluationService:
|
||||
for row in rows:
|
||||
run = {
|
||||
"run_id": row.run_id,
|
||||
"name": self._run_name_from_row(row),
|
||||
"dataset_id": row.dataset_id,
|
||||
"status": row.status,
|
||||
"started_at": format_utc_datetime(row.started_at),
|
||||
@ -669,6 +698,7 @@ class EvaluationService:
|
||||
paged_items = [self._run_item_to_dict(item) for item in details]
|
||||
return {
|
||||
"run_id": row.run_id,
|
||||
"name": self._run_name_from_row(row),
|
||||
"status": row.status,
|
||||
"started_at": format_utc_datetime(row.started_at),
|
||||
"completed_at": format_utc_datetime(row.completed_at),
|
||||
|
||||
@ -150,6 +150,7 @@ class PostgresManager(metaclass=SingletonMeta):
|
||||
"ALTER TABLE IF EXISTS evaluation_datasets ADD COLUMN IF NOT EXISTS created_by VARCHAR(64)",
|
||||
"ALTER TABLE IF EXISTS evaluation_datasets ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ",
|
||||
"ALTER TABLE IF EXISTS evaluation_datasets ADD COLUMN IF NOT EXISTS build_metadata JSONB",
|
||||
"ALTER TABLE IF EXISTS evaluation_runs ADD COLUMN IF NOT EXISTS name VARCHAR(255)",
|
||||
"ALTER TABLE IF EXISTS evaluation_runs ADD COLUMN IF NOT EXISTS metrics JSONB",
|
||||
"ALTER TABLE IF EXISTS evaluation_runs ADD COLUMN IF NOT EXISTS overall_score DOUBLE PRECISION",
|
||||
"ALTER TABLE IF EXISTS evaluation_runs ADD COLUMN IF NOT EXISTS total_items INTEGER",
|
||||
@ -197,6 +198,7 @@ class PostgresManager(metaclass=SingletonMeta):
|
||||
CREATE TABLE IF NOT EXISTS evaluation_runs (
|
||||
id SERIAL PRIMARY KEY,
|
||||
run_id VARCHAR(64) NOT NULL UNIQUE,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
kb_id VARCHAR(80) NOT NULL REFERENCES knowledge_bases(kb_id) ON DELETE CASCADE,
|
||||
dataset_id VARCHAR(64) REFERENCES evaluation_datasets(dataset_id) ON DELETE SET NULL,
|
||||
status VARCHAR(32) DEFAULT 'running',
|
||||
|
||||
@ -241,6 +241,7 @@ class EvaluationRun(Base):
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
run_id = Column(String(64), unique=True, nullable=False, index=True)
|
||||
name = Column(String(255), nullable=False)
|
||||
kb_id = Column(String(80), ForeignKey("knowledge_bases.kb_id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
dataset_id = Column(
|
||||
String(64),
|
||||
|
||||
@ -34,6 +34,7 @@ class GenerateDatasetRequest(BaseModel):
|
||||
|
||||
class RunEvaluationRequest(BaseModel):
|
||||
dataset_id: str = Field(..., min_length=1)
|
||||
name: str | None = Field(default=None, min_length=1, max_length=100)
|
||||
retrieval_config: dict[str, Any] = Field(default_factory=dict, alias="model_config")
|
||||
|
||||
|
||||
@ -176,6 +177,7 @@ async def run_evaluation(kb_id: str, request: RunEvaluationRequest, current_user
|
||||
run_id = await service.run_evaluation(
|
||||
kb_id=kb_id,
|
||||
dataset_id=request.dataset_id,
|
||||
name=request.name,
|
||||
model_config=request.retrieval_config,
|
||||
created_by=current_user.uid,
|
||||
)
|
||||
|
||||
@ -20,9 +20,13 @@ def test_retrieval_metrics_use_metadata_chunk_id():
|
||||
assert metrics["f1@3"] == RetrievalMetrics.f1_score_at_k(["chunk_a", "chunk_b"], ["chunk_b", "chunk_c"], 3)
|
||||
|
||||
|
||||
def test_overall_score_keeps_existing_average_strategy():
|
||||
score = EvaluationMetricsCalculator.calculate_overall_score(
|
||||
[{"recall@1": 1.0, "f1@1": 0.5}], [{"score": 0.25}]
|
||||
)
|
||||
def test_overall_score_uses_retrieval_metrics_only():
|
||||
score = EvaluationMetricsCalculator.calculate_overall_score([{"recall@1": 1.0, "f1@1": 0.5}], [{"score": 0.25}])
|
||||
|
||||
assert score == 0.5
|
||||
assert score == 0.75
|
||||
|
||||
|
||||
def test_overall_score_returns_none_without_retrieval_metrics():
|
||||
score = EvaluationMetricsCalculator.calculate_overall_score([], [{"score": 1.0}])
|
||||
|
||||
assert score is None
|
||||
|
||||
@ -3,13 +3,15 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
|
||||
from yuxi.knowledge.eval import service as eval_service_module
|
||||
from yuxi.knowledge.eval.service import EvaluationService
|
||||
from yuxi.knowledge.eval.service import EvaluationService, build_evaluation_run_name
|
||||
|
||||
|
||||
class FakeEvaluationRepository:
|
||||
def __init__(self):
|
||||
self.created_dataset = None
|
||||
self.updated_dataset = None
|
||||
self.dataset = None
|
||||
self.created_run = None
|
||||
|
||||
async def create_dataset(self, payload):
|
||||
self.created_dataset = payload
|
||||
@ -17,6 +19,12 @@ class FakeEvaluationRepository:
|
||||
async def update_dataset(self, dataset_id, payload):
|
||||
self.updated_dataset = (dataset_id, payload)
|
||||
|
||||
async def get_dataset(self, dataset_id):
|
||||
return self.dataset
|
||||
|
||||
async def create_run(self, payload):
|
||||
self.created_run = payload
|
||||
|
||||
|
||||
class FakeChunkRepository:
|
||||
def __init__(self, indexed_count):
|
||||
@ -26,6 +34,11 @@ class FakeChunkRepository:
|
||||
return self.indexed_count
|
||||
|
||||
|
||||
class FakeKnowledgeBaseRepository:
|
||||
async def get_by_kb_id(self, kb_id):
|
||||
return SimpleNamespace(query_params={"options": {"top_k": 3}})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_dataset_saves_generation_params(monkeypatch):
|
||||
async def fake_enqueue(**kwargs):
|
||||
@ -78,3 +91,43 @@ async def test_generate_dataset_rejects_graph_mode_without_indexed_chunks():
|
||||
)
|
||||
|
||||
assert service.eval_repo.created_dataset is None
|
||||
|
||||
|
||||
def test_build_evaluation_run_name_uses_eval_date_hash_format():
|
||||
name = build_evaluation_run_name(hash_value="abcdef12")
|
||||
|
||||
assert name.startswith("eval-")
|
||||
assert name.endswith("-abcdef")
|
||||
assert len(name.split("-")[1]) == 8
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_evaluation_saves_custom_name(monkeypatch):
|
||||
async def fake_enqueue(**kwargs):
|
||||
return SimpleNamespace(id="task_1")
|
||||
|
||||
monkeypatch.setattr(eval_service_module.tasker, "enqueue", fake_enqueue)
|
||||
repo = FakeEvaluationRepository()
|
||||
repo.dataset = SimpleNamespace(
|
||||
dataset_id="dataset_1",
|
||||
kb_id="db_1",
|
||||
name="dataset",
|
||||
item_count=2,
|
||||
build_metadata={"status": "completed"},
|
||||
)
|
||||
service = EvaluationService()
|
||||
service.eval_repo = repo
|
||||
service.kb_repo = FakeKnowledgeBaseRepository()
|
||||
|
||||
run_id = await service.run_evaluation(
|
||||
kb_id="db_1",
|
||||
dataset_id="dataset_1",
|
||||
name=" 回归评估 ",
|
||||
model_config={"answer_llm": "test:model"},
|
||||
created_by="user_1",
|
||||
)
|
||||
|
||||
assert run_id.startswith("run_")
|
||||
assert repo.created_run["name"] == "回归评估"
|
||||
assert repo.created_run["retrieval_config"]["top_k"] == 3
|
||||
assert repo.created_run["retrieval_config"]["answer_llm"] == "test:model"
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
- 收敛知识库文件解析参数：文件级 `processing_params` 统一保存 `ocr_engine` 与 `ocr_engine_config`，解析阶段直接使用该结构并保留分块参数快照。
|
||||
- 修复知识库文件大小显示为 0 的问题：文件上传时 `file_sizes` 参数未正确传播或历史数据缺失导致 DB 中 `file_size` 为 `None`；新增 `MinIOClient.stat_file/astat_file` 获取文件大小方法，`add_file_record` 在 `size` 缺失时从 MinIO 回补，`_load_metadata` 加载元数据后自动为缺少 `size` 的文件从 MinIO 补全并持久化。
|
||||
- 优化评估基准自动生成：生成任务支持配置队列并发数，默认 10，范围 1-20。
|
||||
- 重梳理知识库评估存储：评估数据集、题目、评估运行和逐题结果统一入库，JSONL 仅作为导入/导出格式；后端和前端 API 统一使用 dataset/run 语义。
|
||||
- 重梳理知识库评估存储：评估数据集、题目、评估运行和逐题结果统一入库，JSONL 仅作为导入/导出格式；后端和前端 API 统一使用 dataset/run 语义；评估运行支持用户命名，历史记录按名称展示，综合评分只聚合检索指标。
|
||||
- 扩展知识库上传来源：添加“从工作区上传”模式，后端将当前用户工作区文件预处理上传到 MinIO，前端沿用现有 `addDocuments` 入库链路提交 MinIO URL、内容哈希和文件大小。
|
||||
- 重构知识库详情页布局：`DatabaseInfo` 改为顶部详情 header + 左侧功能 tab 侧边栏 + 右侧内容区，Milvus 默认进入文件管理，并将检索测试、知识图谱、知识导图、检索配置、RAG 评估和评估基准统一纳入侧边栏导航；只读连接器保留检索测试与检索配置。
|
||||
- 整合知识导图接口：移除独立 mindmap router 与前端 API 模块，思维导图生成、查询和文件列表接口统一收敛到知识库 API 下。
|
||||
|
||||
@ -56,7 +56,7 @@ JSONL 只是导入和导出的交换格式。导入后，系统会把评估数
|
||||
|
||||
## 运行评估
|
||||
|
||||
在知识库详情页左侧边栏，「评估基准」Tab 用于管理评估数据集，「RAG 评估」Tab 用于运行评估并查看结果。在「RAG 评估」中选择评估数据集后配置：
|
||||
在知识库详情页左侧边栏，「评估基准」Tab 用于管理评估数据集，「RAG 评估」Tab 用于运行评估并查看结果。在「RAG 评估」中填写评估名称、选择评估数据集后配置：
|
||||
|
||||
1. **答案生成模型**（可选）：基于检索到的文档块生成答案
|
||||
2. **评判模型**（可选）：评估生成答案与标准答案的一致性
|
||||
|
||||
@ -38,6 +38,15 @@
|
||||
</div>
|
||||
|
||||
<div class="dropdown-model-fields">
|
||||
<a-form-item label="评估名称">
|
||||
<a-input
|
||||
v-model:value="configForm.name"
|
||||
placeholder="请输入评估名称"
|
||||
:maxlength="100"
|
||||
show-count
|
||||
/>
|
||||
</a-form-item>
|
||||
|
||||
<a-form-item label="评估基准">
|
||||
<div class="dropdown-benchmark-row">
|
||||
<a-select
|
||||
@ -126,12 +135,13 @@
|
||||
</div>
|
||||
<div class="last-evaluation-info">
|
||||
<div class="last-evaluation-title">
|
||||
{{ getDatasetName(latestEvaluation.dataset_id) }}
|
||||
{{ getRunName(latestEvaluation) }}
|
||||
<a-tag :color="getStatusColor(latestEvaluation.status)" :bordered="false">
|
||||
{{ getStatusText(latestEvaluation.status) }}
|
||||
</a-tag>
|
||||
</div>
|
||||
<div class="last-evaluation-meta">
|
||||
{{ getDatasetName(latestEvaluation.dataset_id) }} ·
|
||||
{{ formatTime(latestEvaluation.started_at) }} ·
|
||||
<button
|
||||
v-if="latestEvaluation.status === 'completed'"
|
||||
@ -244,9 +254,7 @@
|
||||
<div v-if="resultModalVisible" class="evaluation-detail-overlay">
|
||||
<div class="evaluation-detail-panel">
|
||||
<div class="evaluation-detail-titlebar">
|
||||
<div class="evaluation-detail-title">
|
||||
评估结果 - {{ selectedResult?.run_id?.slice(0, 8) || '' }}
|
||||
</div>
|
||||
<div class="evaluation-detail-title">评估结果 - {{ getRunName(selectedResult) }}</div>
|
||||
<a-button
|
||||
type="text"
|
||||
size="small"
|
||||
@ -487,8 +495,29 @@ const isDatasetCompleted = (dataset) =>
|
||||
|
||||
const latestEvaluation = computed(() => evaluationHistory.value[0] || null)
|
||||
|
||||
const createDefaultNameHash = () => {
|
||||
const cryptoApi = globalThis.crypto
|
||||
if (cryptoApi?.getRandomValues) {
|
||||
const values = new Uint32Array(1)
|
||||
cryptoApi.getRandomValues(values)
|
||||
return values[0].toString(16).padStart(6, '0').slice(0, 6)
|
||||
}
|
||||
return Math.floor(Math.random() * 0xffffff)
|
||||
.toString(16)
|
||||
.padStart(6, '0')
|
||||
}
|
||||
|
||||
const buildDefaultEvaluationName = () => {
|
||||
const now = new Date()
|
||||
const year = now.getFullYear()
|
||||
const month = String(now.getMonth() + 1).padStart(2, '0')
|
||||
const day = String(now.getDate()).padStart(2, '0')
|
||||
return `eval-${year}${month}${day}-${createDefaultNameHash()}`
|
||||
}
|
||||
|
||||
// 评估配置表单（使用知识库默认配置）
|
||||
const configForm = reactive({
|
||||
name: buildDefaultEvaluationName(),
|
||||
answer_llm: '', // 答案生成模型
|
||||
judge_llm: '' // 评判模型
|
||||
})
|
||||
@ -591,11 +620,12 @@ const withResizableTitle = (column) => ({
|
||||
|
||||
const historyColumns = [
|
||||
{
|
||||
title: '开始时间',
|
||||
dataIndex: 'started_at',
|
||||
key: 'started_at',
|
||||
width: 160,
|
||||
customRender: ({ record }) => formatTime(record.started_at)
|
||||
title: '评估名称',
|
||||
dataIndex: 'name',
|
||||
key: 'name',
|
||||
width: 180,
|
||||
ellipsis: true,
|
||||
customRender: ({ record }) => getRunName(record)
|
||||
},
|
||||
{
|
||||
title: '评估基准',
|
||||
@ -820,16 +850,22 @@ const startEvaluation = async () => {
|
||||
const judgeModel = selectedDataset.value.has_gold_answers ? configForm.judge_llm : ''
|
||||
const hasAnswerModel = !!answerModel
|
||||
const hasJudgeModel = !!judgeModel
|
||||
const runName = configForm.name.trim()
|
||||
|
||||
if (hasAnswerModel !== hasJudgeModel) {
|
||||
message.warning('生成模型和评估模型必须同时选择或者同时不选择')
|
||||
return
|
||||
}
|
||||
if (!runName) {
|
||||
message.warning('请输入评估名称')
|
||||
return
|
||||
}
|
||||
|
||||
startingEvaluation.value = true
|
||||
|
||||
const params = {
|
||||
dataset_id: selectedDataset.value.dataset_id,
|
||||
name: runName,
|
||||
model_config: {
|
||||
answer_llm: answerModel,
|
||||
judge_llm: judgeModel
|
||||
@ -842,6 +878,7 @@ const startEvaluation = async () => {
|
||||
if (response.message === 'success') {
|
||||
message.success('评估任务已开始')
|
||||
evaluationDropdownOpen.value = false
|
||||
configForm.name = buildDefaultEvaluationName()
|
||||
loadEvaluationHistory()
|
||||
taskerStore.loadTasks()
|
||||
} else {
|
||||
@ -945,6 +982,7 @@ const viewResults = async (runId) => {
|
||||
// 从历史记录中找到对应的任务信息，如果没有则使用API返回的数据
|
||||
selectedResult.value = evaluationHistory.value.find((r) => r.run_id === runId) || {
|
||||
run_id: resultData.run_id,
|
||||
name: resultData.name,
|
||||
status: resultData.status,
|
||||
started_at: resultData.started_at,
|
||||
completed_at: resultData.completed_at,
|
||||
@ -1015,6 +1053,9 @@ const getDatasetName = (datasetId) => {
|
||||
return benchmark ? benchmark.name : datasetId?.slice(0, 8) || '-'
|
||||
}
|
||||
|
||||
const getRunName = (record) =>
|
||||
record?.name || record?.run_name || record?.run_id?.slice(0, 8) || '-'
|
||||
|
||||
const getRecall10 = (record) => record?.metrics?.['recall@10']
|
||||
|
||||
const formatPercent = (value, digits = 1) => {
|
||||
@ -1179,6 +1220,12 @@ const formatDuration = (seconds) => {
|
||||
|
||||
watch(evaluationHistory, syncEvaluationRefresh, { deep: true })
|
||||
|
||||
watch(evaluationDropdownOpen, (open) => {
|
||||
if (open) {
|
||||
configForm.name = buildDefaultEvaluationName()
|
||||
}
|
||||
})
|
||||
|
||||
// 组件挂载时加载数据
|
||||
onMounted(() => {
|
||||
loadDatasets()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user