From 70b76f1e6d922970e998da0b03842b7e8ace646f Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Thu, 4 Jun 2026 13:39:09 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E8=AF=84=E4=BC=B0?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E5=91=BD=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../package/yuxi/knowledge/eval/evaluator.py | 2 +- .../package/yuxi/knowledge/eval/metrics.py | 21 ++---- .../package/yuxi/knowledge/eval/service.py | 34 +++++++++- .../package/yuxi/storage/postgres/manager.py | 2 + .../yuxi/storage/postgres/models_knowledge.py | 1 + .../server/routers/knowledge_eval_router.py | 2 + .../test/unit/knowledge/eval/test_metrics.py | 14 ++-- .../knowledge/eval/test_service_generation.py | 55 +++++++++++++++- docs/develop-guides/roadmap.md | 2 +- docs/intro/evaluation.md | 2 +- web/src/components/RAGEvaluationTab.vue | 65 ++++++++++++++++--- 11 files changed, 164 insertions(+), 36 deletions(-) diff --git a/backend/package/yuxi/knowledge/eval/evaluator.py b/backend/package/yuxi/knowledge/eval/evaluator.py index ce9ec4e3..a8177937 100644 --- a/backend/package/yuxi/knowledge/eval/evaluator.py +++ b/backend/package/yuxi/knowledge/eval/evaluator.py @@ -118,7 +118,7 @@ def aggregate_metrics( answer_metrics_list: list[dict[str, Any]], *, include_overall_score: bool = False, -) -> tuple[dict[str, Any], float]: +) -> tuple[dict[str, Any], float | None]: overall_metrics = {} if retrieval_metrics_list: diff --git a/backend/package/yuxi/knowledge/eval/metrics.py b/backend/package/yuxi/knowledge/eval/metrics.py index b8342584..efd9499b 100644 --- a/backend/package/yuxi/knowledge/eval/metrics.py +++ b/backend/package/yuxi/knowledge/eval/metrics.py @@ -131,24 +131,13 @@ class EvaluationMetricsCalculator: @staticmethod def calculate_overall_score( retrieval_metrics_list: list[dict[str, float]], answer_metrics_list: list[dict[str, Any]] - ) -> float: - """计算整体平均分""" - total_score = 0.0 - count = 0 - - # 简单的平均策略：将所有retrieval metric的值和answer metric的score一起平均 - # 用户可能希望分开看，但calculate_overall_score返回一个单值。 + ) -> float | None: + """计算整体平均分，只聚合检索指标。""" + retrieval_scores = [] # 计算检索平均分 for m in retrieval_metrics_list: if m: - total_score += sum(m.values()) / len(m) - count += 1 + retrieval_scores.append(sum(m.values()) / len(m)) - # 计算答案平均分 - for m in answer_metrics_list: - if "score" in m: - total_score += m["score"] - count += 1 - - return total_score / count if count > 0 else 0.0 + return sum(retrieval_scores) / len(retrieval_scores) if retrieval_scores else None diff --git a/backend/package/yuxi/knowledge/eval/service.py b/backend/package/yuxi/knowledge/eval/service.py index 4956eef7..4d3912a4 100644 --- a/backend/package/yuxi/knowledge/eval/service.py +++ b/backend/package/yuxi/knowledge/eval/service.py @@ -20,6 +20,14 @@ from yuxi.utils import logger from yuxi.utils.datetime_utils import format_utc_datetime, utc_now_naive +def build_evaluation_run_name(started_at=None, hash_value: str | None = None) -> str: + date_part = (started_at or utc_now_naive()).strftime("%Y%m%d") + hash_part = re.sub(r"[^a-fA-F0-9]", "", hash_value or uuid.uuid4().hex).lower()[:6] + if len(hash_part) < 6: + hash_part = (hash_part + uuid.uuid4().hex)[:6] + return f"eval-{date_part}-{hash_part}" + + class EvaluationService: """RAG评估服务""" @@ -70,6 +78,18 @@ class EvaluationService: metrics.get(key, 1.0) < 0.3 for key in metrics if key.startswith("recall@") ) + def _normalize_run_name(self, name: str | None, run_id: str) -> str: + run_name = (name or "").strip() + if run_name: + return run_name + return build_evaluation_run_name(hash_value=run_id.removeprefix("run_")) + + def _run_name_from_row(self, row) -> str: + name = (getattr(row, "name", None) or "").strip() + if name: + return name + return build_evaluation_run_name(row.started_at, hash_value=row.run_id.removeprefix("run_")) + async def _sync_dataset_build_metadata(self, row) -> None: metadata = dict(row.build_metadata or {}) if metadata.get("source") != "generated" or metadata.get("status") not in {"pending", "running"}: @@ -419,10 +439,16 @@ class EvaluationService: raise async def run_evaluation( - self, kb_id: str, dataset_id: str, model_config: dict[str, Any] = None, created_by: str = "system" + self, + kb_id: str, + dataset_id: str, + name: str | None = None, + model_config: dict[str, Any] = None, + created_by: str = "system", ) -> str: try: run_id = f"run_{uuid.uuid4().hex[:8]}" + run_name = self._normalize_run_name(name, run_id) dataset_row = await self.eval_repo.get_dataset(dataset_id) if dataset_row is None or dataset_row.kb_id != kb_id: raise ValueError("Dataset not found") @@ -448,6 +474,7 @@ class EvaluationService: await self.eval_repo.create_run( { "run_id": run_id, + "name": run_name, "kb_id": kb_id, "dataset_id": dataset_id, "status": "running", @@ -463,10 +490,11 @@ class EvaluationService: ) await tasker.enqueue( - name=f"RAG评估({dataset_row.name})", + name=f"RAG评估({run_name})", task_type="rag_evaluation", payload={ "run_id": run_id, + "name": run_name, "kb_id": kb_id, "dataset_id": dataset_id, "retrieval_config": retrieval_config, @@ -614,6 +642,7 @@ class EvaluationService: for row in rows: run = { "run_id": row.run_id, + "name": self._run_name_from_row(row), "dataset_id": row.dataset_id, "status": row.status, "started_at": format_utc_datetime(row.started_at), @@ -669,6 +698,7 @@ class EvaluationService: paged_items = [self._run_item_to_dict(item) for item in details] return { "run_id": row.run_id, + "name": self._run_name_from_row(row), "status": row.status, "started_at": format_utc_datetime(row.started_at), "completed_at": format_utc_datetime(row.completed_at), diff --git a/backend/package/yuxi/storage/postgres/manager.py b/backend/package/yuxi/storage/postgres/manager.py index a40ea5fd..09414a64 100644 --- a/backend/package/yuxi/storage/postgres/manager.py +++ b/backend/package/yuxi/storage/postgres/manager.py @@ -150,6 +150,7 @@ class PostgresManager(metaclass=SingletonMeta): "ALTER TABLE IF EXISTS evaluation_datasets ADD COLUMN IF NOT EXISTS created_by VARCHAR(64)", "ALTER TABLE IF EXISTS evaluation_datasets ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ", "ALTER TABLE IF EXISTS evaluation_datasets ADD COLUMN IF NOT EXISTS build_metadata JSONB", + "ALTER TABLE IF EXISTS evaluation_runs ADD COLUMN IF NOT EXISTS name VARCHAR(255)", "ALTER TABLE IF EXISTS evaluation_runs ADD COLUMN IF NOT EXISTS metrics JSONB", "ALTER TABLE IF EXISTS evaluation_runs ADD COLUMN IF NOT EXISTS overall_score DOUBLE PRECISION", "ALTER TABLE IF EXISTS evaluation_runs ADD COLUMN IF NOT EXISTS total_items INTEGER", @@ -197,6 +198,7 @@ class PostgresManager(metaclass=SingletonMeta): CREATE TABLE IF NOT EXISTS evaluation_runs ( id SERIAL PRIMARY KEY, run_id VARCHAR(64) NOT NULL UNIQUE, + name VARCHAR(255) NOT NULL, kb_id VARCHAR(80) NOT NULL REFERENCES knowledge_bases(kb_id) ON DELETE CASCADE, dataset_id VARCHAR(64) REFERENCES evaluation_datasets(dataset_id) ON DELETE SET NULL, status VARCHAR(32) DEFAULT 'running', diff --git a/backend/package/yuxi/storage/postgres/models_knowledge.py b/backend/package/yuxi/storage/postgres/models_knowledge.py index ef78cc06..486f506f 100644 --- a/backend/package/yuxi/storage/postgres/models_knowledge.py +++ b/backend/package/yuxi/storage/postgres/models_knowledge.py @@ -241,6 +241,7 @@ class EvaluationRun(Base): id = Column(Integer, primary_key=True, autoincrement=True) run_id = Column(String(64), unique=True, nullable=False, index=True) + name = Column(String(255), nullable=False) kb_id = Column(String(80), ForeignKey("knowledge_bases.kb_id", ondelete="CASCADE"), nullable=False, index=True) dataset_id = Column( String(64), diff --git a/backend/server/routers/knowledge_eval_router.py b/backend/server/routers/knowledge_eval_router.py index f3fbedb1..1285d669 100644 --- a/backend/server/routers/knowledge_eval_router.py +++ b/backend/server/routers/knowledge_eval_router.py @@ -34,6 +34,7 @@ class GenerateDatasetRequest(BaseModel): class RunEvaluationRequest(BaseModel): dataset_id: str = Field(..., min_length=1) + name: str | None = Field(default=None, min_length=1, max_length=100) retrieval_config: dict[str, Any] = Field(default_factory=dict, alias="model_config") @@ -176,6 +177,7 @@ async def run_evaluation(kb_id: str, request: RunEvaluationRequest, current_user run_id = await service.run_evaluation( kb_id=kb_id, dataset_id=request.dataset_id, + name=request.name, model_config=request.retrieval_config, created_by=current_user.uid, ) diff --git a/backend/test/unit/knowledge/eval/test_metrics.py b/backend/test/unit/knowledge/eval/test_metrics.py index f5ab7503..17cc724c 100644 --- a/backend/test/unit/knowledge/eval/test_metrics.py +++ b/backend/test/unit/knowledge/eval/test_metrics.py @@ -20,9 +20,13 @@ def test_retrieval_metrics_use_metadata_chunk_id(): assert metrics["f1@3"] == RetrievalMetrics.f1_score_at_k(["chunk_a", "chunk_b"], ["chunk_b", "chunk_c"], 3) -def test_overall_score_keeps_existing_average_strategy(): - score = EvaluationMetricsCalculator.calculate_overall_score( - [{"recall@1": 1.0, "f1@1": 0.5}], [{"score": 0.25}] - ) +def test_overall_score_uses_retrieval_metrics_only(): + score = EvaluationMetricsCalculator.calculate_overall_score([{"recall@1": 1.0, "f1@1": 0.5}], [{"score": 0.25}]) - assert score == 0.5 + assert score == 0.75 + + +def test_overall_score_returns_none_without_retrieval_metrics(): + score = EvaluationMetricsCalculator.calculate_overall_score([], [{"score": 1.0}]) + + assert score is None diff --git a/backend/test/unit/knowledge/eval/test_service_generation.py b/backend/test/unit/knowledge/eval/test_service_generation.py index 3601948d..f0474de5 100644 --- a/backend/test/unit/knowledge/eval/test_service_generation.py +++ b/backend/test/unit/knowledge/eval/test_service_generation.py @@ -3,13 +3,15 @@ from types import SimpleNamespace import pytest from yuxi.knowledge.eval import service as eval_service_module -from yuxi.knowledge.eval.service import EvaluationService +from yuxi.knowledge.eval.service import EvaluationService, build_evaluation_run_name class FakeEvaluationRepository: def __init__(self): self.created_dataset = None self.updated_dataset = None + self.dataset = None + self.created_run = None async def create_dataset(self, payload): self.created_dataset = payload @@ -17,6 +19,12 @@ class FakeEvaluationRepository: async def update_dataset(self, dataset_id, payload): self.updated_dataset = (dataset_id, payload) + async def get_dataset(self, dataset_id): + return self.dataset + + async def create_run(self, payload): + self.created_run = payload + class FakeChunkRepository: def __init__(self, indexed_count): @@ -26,6 +34,11 @@ class FakeChunkRepository: return self.indexed_count +class FakeKnowledgeBaseRepository: + async def get_by_kb_id(self, kb_id): + return SimpleNamespace(query_params={"options": {"top_k": 3}}) + + @pytest.mark.asyncio async def test_generate_dataset_saves_generation_params(monkeypatch): async def fake_enqueue(**kwargs): @@ -78,3 +91,43 @@ async def test_generate_dataset_rejects_graph_mode_without_indexed_chunks(): ) assert service.eval_repo.created_dataset is None + + +def test_build_evaluation_run_name_uses_eval_date_hash_format(): + name = build_evaluation_run_name(hash_value="abcdef12") + + assert name.startswith("eval-") + assert name.endswith("-abcdef") + assert len(name.split("-")[1]) == 8 + + +@pytest.mark.asyncio +async def test_run_evaluation_saves_custom_name(monkeypatch): + async def fake_enqueue(**kwargs): + return SimpleNamespace(id="task_1") + + monkeypatch.setattr(eval_service_module.tasker, "enqueue", fake_enqueue) + repo = FakeEvaluationRepository() + repo.dataset = SimpleNamespace( + dataset_id="dataset_1", + kb_id="db_1", + name="dataset", + item_count=2, + build_metadata={"status": "completed"}, + ) + service = EvaluationService() + service.eval_repo = repo + service.kb_repo = FakeKnowledgeBaseRepository() + + run_id = await service.run_evaluation( + kb_id="db_1", + dataset_id="dataset_1", + name=" 回归评估 ", + model_config={"answer_llm": "test:model"}, + created_by="user_1", + ) + + assert run_id.startswith("run_") + assert repo.created_run["name"] == "回归评估" + assert repo.created_run["retrieval_config"]["top_k"] == 3 + assert repo.created_run["retrieval_config"]["answer_llm"] == "test:model" diff --git a/docs/develop-guides/roadmap.md b/docs/develop-guides/roadmap.md index 4401ce24..3b58b3b3 100644 --- a/docs/develop-guides/roadmap.md +++ b/docs/develop-guides/roadmap.md @@ -60,7 +60,7 @@ - 收敛知识库文件解析参数：文件级 `processing_params` 统一保存 `ocr_engine` 与 `ocr_engine_config`，解析阶段直接使用该结构并保留分块参数快照。 - 修复知识库文件大小显示为 0 的问题：文件上传时 `file_sizes` 参数未正确传播或历史数据缺失导致 DB 中 `file_size` 为 `None`；新增 `MinIOClient.stat_file/astat_file` 获取文件大小方法，`add_file_record` 在 `size` 缺失时从 MinIO 回补，`_load_metadata` 加载元数据后自动为缺少 `size` 的文件从 MinIO 补全并持久化。 - 优化评估基准自动生成：生成任务支持配置队列并发数，默认 10，范围 1-20。 -- 重梳理知识库评估存储：评估数据集、题目、评估运行和逐题结果统一入库，JSONL 仅作为导入/导出格式；后端和前端 API 统一使用 dataset/run 语义。 +- 重梳理知识库评估存储：评估数据集、题目、评估运行和逐题结果统一入库，JSONL 仅作为导入/导出格式；后端和前端 API 统一使用 dataset/run 语义；评估运行支持用户命名，历史记录按名称展示，综合评分只聚合检索指标。 - 扩展知识库上传来源：添加“从工作区上传”模式，后端将当前用户工作区文件预处理上传到 MinIO，前端沿用现有 `addDocuments` 入库链路提交 MinIO URL、内容哈希和文件大小。 - 重构知识库详情页布局：`DatabaseInfo` 改为顶部详情 header + 左侧功能 tab 侧边栏 + 右侧内容区，Milvus 默认进入文件管理，并将检索测试、知识图谱、知识导图、检索配置、RAG 评估和评估基准统一纳入侧边栏导航；只读连接器保留检索测试与检索配置。 - 整合知识导图接口：移除独立 mindmap router 与前端 API 模块，思维导图生成、查询和文件列表接口统一收敛到知识库 API 下。 diff --git a/docs/intro/evaluation.md b/docs/intro/evaluation.md index adfa2cb9..890f3cdf 100644 --- a/docs/intro/evaluation.md +++ b/docs/intro/evaluation.md @@ -56,7 +56,7 @@ JSONL 只是导入和导出的交换格式。导入后，系统会把评估数 ## 运行评估 -在知识库详情页左侧边栏，「评估基准」Tab 用于管理评估数据集，「RAG 评估」Tab 用于运行评估并查看结果。在「RAG 评估」中选择评估数据集后配置： +在知识库详情页左侧边栏，「评估基准」Tab 用于管理评估数据集，「RAG 评估」Tab 用于运行评估并查看结果。在「RAG 评估」中填写评估名称、选择评估数据集后配置： 1. **答案生成模型**（可选）：基于检索到的文档块生成答案 2. **评判模型**（可选）：评估生成答案与标准答案的一致性 diff --git a/web/src/components/RAGEvaluationTab.vue b/web/src/components/RAGEvaluationTab.vue index 879450e9..47880702 100644 --- a/web/src/components/RAGEvaluationTab.vue +++ b/web/src/components/RAGEvaluationTab.vue @@ -38,6 +38,15 @@
+ + + +
- {{ getDatasetName(latestEvaluation.dataset_id) }} + {{ getRunName(latestEvaluation) }} {{ getStatusText(latestEvaluation.status) }}
+ {{ getDatasetName(latestEvaluation.dataset_id) }} · {{ formatTime(latestEvaluation.started_at) }} ·