diff --git a/backend/package/yuxi/knowledge/eval/benchmark_generation.py b/backend/package/yuxi/knowledge/eval/benchmark_generation.py index cf702da4..4b34a85c 100644 --- a/backend/package/yuxi/knowledge/eval/benchmark_generation.py +++ b/backend/package/yuxi/knowledge/eval/benchmark_generation.py @@ -1,13 +1,11 @@ import json -import math import random from collections.abc import AsyncIterator, Callable from typing import Any import json_repair -from yuxi import config -from yuxi.models import select_embedding_model, select_model +from yuxi.models import select_model from yuxi.utils import logger @@ -36,26 +34,59 @@ def clamp_neighbors_count(neighbors_count: int) -> int: return min(max(neighbors_count, 0), 10) -def cosine_similarity(a: list[float], b: list[float], na: float, nb: float) -> float: - s = 0.0 - for i in range(len(a)): - s += a[i] * b[i] - return s / (na * nb) +def _is_anchor_chunk(candidate: dict[str, Any], anchor_chunk: dict[str, Any]) -> bool: + metadata = candidate.get("metadata") or {} + candidate_id = metadata.get("chunk_id") + if candidate_id is not None and str(candidate_id) == str(anchor_chunk.get("id")): + return True + + candidate_file_id = metadata.get("file_id") + candidate_chunk_index = metadata.get("chunk_index") + return candidate_file_id == anchor_chunk.get("file_id") and candidate_chunk_index == anchor_chunk.get("chunk_index") -def select_neighbor_indices( - anchor_idx: int, embeddings: list[list[float]], norms: list[float], neighbors_count: int -) -> list[int]: - anchor_embedding = embeddings[anchor_idx] - anchor_norm = norms[anchor_idx] - sims = [] - for idx in range(len(embeddings)): - if idx == anchor_idx: +async def select_neighbor_chunks_by_kb_query( + *, kb_instance: Any, db_id: str, anchor_chunk: dict[str, Any], neighbors_count: int +) -> list[dict[str, Any]]: + if neighbors_count <= 0: + return [] + + anchor_content = anchor_chunk.get("content", "") + if not anchor_content: + return [] + + candidates = await kb_instance.aquery( + anchor_content, + db_id, + search_mode="vector", + final_top_k=neighbors_count + 3, + use_reranker=False, + similarity_threshold=0.0, + ) + + chunks = [] + for candidate in candidates: + if _is_anchor_chunk(candidate, anchor_chunk): continue - score = cosine_similarity(anchor_embedding, embeddings[idx], anchor_norm, norms[idx]) - sims.append((idx, score)) - sims.sort(key=lambda x: x[1], reverse=True) - return [idx for idx, _ in sims[:neighbors_count]] + + metadata = candidate.get("metadata") or {} + chunk_id = metadata.get("chunk_id") + content = candidate.get("content", "") + if not chunk_id or not content: + continue + + chunks.append( + { + "id": str(chunk_id), + "content": content, + "file_id": metadata.get("file_id"), + "chunk_index": metadata.get("chunk_index"), + } + ) + if len(chunks) >= neighbors_count: + break + + return chunks def build_benchmark_generation_prompt(ctx_items: list[tuple[str, str]]) -> str: @@ -74,7 +105,6 @@ async def iter_generated_benchmark_items( db_id: str, count: int, neighbors_count: int, - embedding_model_id: str, llm_model_spec: Any, progress_cb: Callable[[int, str], Any] | None = None, ) -> AsyncIterator[dict[str, Any]]: @@ -86,31 +116,25 @@ async def iter_generated_benchmark_items( raise ValueError("No chunks found in knowledge base") if progress_cb: - await progress_cb(15, "向量化") + await progress_cb(15, "准备生成样本") - contents = [chunk["content"] for chunk in all_chunks] - embed_model = select_embedding_model(embedding_model_id) - batch_size = int(getattr(embed_model, "batch_size", 40) or 40) - if embedding_model_id in config.embed_model_names: - batch_size = config.embed_model_names[embedding_model_id].batch_size - - embeddings = await embed_model.abatch_encode(contents, batch_size=batch_size) - norms = [math.sqrt(sum(x * x for x in vec)) or 1.0 for vec in embeddings] llm = select_model(model_spec=llm_model_spec) - neighbors_count = clamp_neighbors_count(neighbors_count) + context_count = max(clamp_neighbors_count(neighbors_count), 1) generated = 0 attempts = 0 max_attempts = max(count * 5, 50) - if progress_cb: - await progress_cb(0, "准备生成样本") - while generated < count and attempts < max_attempts: attempts += 1 - anchor_idx = random.randrange(len(all_chunks)) - neighbor_indices = select_neighbor_indices(anchor_idx, embeddings, norms, neighbors_count) - ctx_items = [(all_chunks[anchor_idx]["id"], all_chunks[anchor_idx]["content"])] - ctx_items.extend((all_chunks[idx]["id"], all_chunks[idx]["content"]) for idx in neighbor_indices) + anchor_chunk = all_chunks[random.randrange(len(all_chunks))] + neighbor_chunks = await select_neighbor_chunks_by_kb_query( + kb_instance=kb_instance, + db_id=db_id, + anchor_chunk=anchor_chunk, + neighbors_count=context_count - 1, + ) + ctx_chunks = [anchor_chunk] + neighbor_chunks + ctx_items = [(chunk["id"], chunk["content"]) for chunk in ctx_chunks] allowed_ids = {cid for cid, _ in ctx_items} try: diff --git a/backend/package/yuxi/knowledge/implementations/milvus.py b/backend/package/yuxi/knowledge/implementations/milvus.py index 5ba23cf2..5f8d9875 100644 --- a/backend/package/yuxi/knowledge/implementations/milvus.py +++ b/backend/package/yuxi/knowledge/implementations/milvus.py @@ -32,6 +32,7 @@ from yuxi.utils.datetime_utils import utc_isoformat MILVUS_AVAILABLE = True CONTENT_SPARSE_FIELD = "content_sparse" CONTENT_ANALYZER_PARAMS = {"type": "chinese"} +VECTOR_METRIC_TYPE = "COSINE" class MilvusKB(KnowledgeBase): @@ -189,7 +190,7 @@ class MilvusKB(KnowledgeBase): collection = Collection(name=collection_name, schema=schema, using=self.connection_alias) # 创建索引 - index_params = {"metric_type": "COSINE", "index_type": "IVF_FLAT", "params": {"nlist": 1024}} + index_params = {"metric_type": VECTOR_METRIC_TYPE, "index_type": "IVF_FLAT", "params": {"nlist": 1024}} collection.create_index("embedding", index_params) sparse_index_params = { "metric_type": "BM25", @@ -572,7 +573,7 @@ class MilvusKB(KnowledgeBase): final_top_k = int(merged_kwargs.get("final_top_k", 10)) final_top_k = max(final_top_k, 1) similarity_threshold = float(merged_kwargs.get("similarity_threshold", 0.2)) - metric_type = merged_kwargs.get("metric_type", "COSINE") + metric_type = VECTOR_METRIC_TYPE include_distances = bool(merged_kwargs.get("include_distances", True)) search_mode = str(merged_kwargs.get("search_mode", "vector")).lower() if search_mode not in {"vector", "keyword", "hybrid"}: @@ -615,7 +616,7 @@ class MilvusKB(KnowledgeBase): if results and len(results) > 0 and len(results[0]) > 0: for hit in results[0]: - similarity = hit.distance if metric_type == "COSINE" else 1 / (1 + hit.distance) + similarity = hit.distance if metric_type == VECTOR_METRIC_TYPE else 1 / (1 + hit.distance) if similarity < similarity_threshold: continue @@ -938,18 +939,6 @@ class MilvusKB(KnowledgeBase): "default": True, "description": "在结果中显示相似度分数", }, - { - "key": "metric_type", - "label": "距离度量类型", - "type": "select", - "default": "COSINE", - "options": [ - {"value": "COSINE", "label": "余弦相似度", "description": "适合文本语义相似度"}, - {"value": "L2", "label": "欧几里得距离", "description": "适合数值型数据"}, - {"value": "IP", "label": "内积", "description": "适合标准化向量"}, - ], - "description": "向量相似度计算方法", - }, { "key": "use_reranker", "label": "启用重排序", diff --git a/backend/package/yuxi/services/evaluation_service.py b/backend/package/yuxi/services/evaluation_service.py index e3163a62..bff59ac4 100644 --- a/backend/package/yuxi/services/evaluation_service.py +++ b/backend/package/yuxi/services/evaluation_service.py @@ -297,25 +297,17 @@ class EvaluationService: name = payload.get("name", "自动生成评估基准") description = payload.get("description", "") count = int(payload.get("count", 10)) - neighbors_count = int(payload.get("neighbors_count", 0)) - embedding_model_id = payload.get("embedding_model_id") - llm_model_spec = payload.get("llm_model_spec") or (payload.get("llm_config") or {}).get("model_spec") + neighbors_count = int(payload.get("neighbors_count", 1)) + llm_model_spec = payload.get("llm_model_spec") kb_instance = await knowledge_base.aget_kb(db_id) if not kb_instance: await context.set_message("知识库不存在") raise ValueError("Knowledge Base not found") - if kb_instance.kb_type == "lightrag": - await context.set_message("暂不支持该类型知识库生成评估基准") + if kb_instance.kb_type != "milvus": + await context.set_message("仅支持 commonrag/Milvus 类型知识库生成评估基准") raise ValueError("Unsupported KB type for benchmark generation") - db_meta = kb_instance.databases_meta.get(db_id, {}) - embed_info = db_meta.get("embed_info", {}) - if not embedding_model_id: - embedding_model_id = embed_info.get("name") or embed_info.get("model") or "" - if not embedding_model_id: - raise ValueError("Embedding model not specified") - benchmark_id = f"benchmark_{uuid.uuid4().hex[:8]}" bench_dir = await self._get_benchmark_dir(db_id) data_file_path = os.path.join(bench_dir, f"{benchmark_id}.jsonl") @@ -328,7 +320,6 @@ class EvaluationService: db_id=db_id, count=count, neighbors_count=neighbors_count, - embedding_model_id=embedding_model_id, llm_model_spec=llm_model_spec, progress_cb=context.set_progress, ): diff --git a/backend/test/unit/knowledge/eval/test_benchmark_generation.py b/backend/test/unit/knowledge/eval/test_benchmark_generation.py index 49cf51ca..508f8bdb 100644 --- a/backend/test/unit/knowledge/eval/test_benchmark_generation.py +++ b/backend/test/unit/knowledge/eval/test_benchmark_generation.py @@ -1,15 +1,17 @@ import os +from types import SimpleNamespace import pytest os.environ.setdefault("OPENAI_API_KEY", "test-key") +from yuxi.knowledge.eval import benchmark_generation from yuxi.knowledge.eval.benchmark_generation import ( build_benchmark_generation_prompt, clamp_neighbors_count, collect_kb_chunks, - cosine_similarity, - select_neighbor_indices, + iter_generated_benchmark_items, + select_neighbor_chunks_by_kb_query, ) @@ -27,19 +29,48 @@ class FakeKnowledgeBase: } +class FakeGenerationKnowledgeBase: + files_meta = {"file_a": {"database_id": "db_1"}} + + def __init__(self, query_results=None): + self.query_results = query_results or [] + self.query_calls = [] + + async def get_file_content(self, db_id, fid): + return { + "lines": [ + {"id": "anchor_chunk", "content": "anchor content", "chunk_order_index": 0}, + ] + } + + async def aquery(self, query_text, db_id, **kwargs): + self.query_calls.append({"query_text": query_text, "db_id": db_id, **kwargs}) + return self.query_results + + +class FakeLlm: + def __init__(self, gold_chunk_id="anchor_chunk"): + self.gold_chunk_id = gold_chunk_id + self.prompts = [] + + async def call(self, prompt, stream): + self.prompts.append(prompt) + return SimpleNamespace( + content=('{"query":"问题","gold_answer":"答案","gold_chunk_ids":["' + self.gold_chunk_id + '"]}') + ) + + +class NoQueryKnowledgeBase(FakeGenerationKnowledgeBase): + async def aquery(self, query_text, db_id, **kwargs): + raise AssertionError("neighbors_count=1 时不应调用 aquery") + + def test_clamp_neighbors_count(): assert clamp_neighbors_count(-1) == 0 assert clamp_neighbors_count(3) == 3 assert clamp_neighbors_count(11) == 10 -def test_select_neighbor_indices_orders_by_cosine_similarity(): - embeddings = [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]] - norms = [1.0, cosine_similarity(embeddings[1], embeddings[1], 1.0, 1.0) ** 0.5, 1.0] - - assert select_neighbor_indices(0, embeddings, norms, 1) == [1] - - def test_build_benchmark_generation_prompt_contains_required_schema(): prompt = build_benchmark_generation_prompt([("chunk_1", "片段内容")]) @@ -52,3 +83,108 @@ async def test_collect_kb_chunks_filters_database_id(): chunks = await collect_kb_chunks(FakeKnowledgeBase(), "db_1") assert chunks == [{"id": "file_a_chunk", "content": "内容", "file_id": "file_a", "chunk_index": 0}] + + +@pytest.mark.asyncio +async def test_iter_generated_benchmark_items_with_one_chunk_does_not_query(monkeypatch): + fake_llm = FakeLlm() + monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm) + + items = [ + item + async for item in iter_generated_benchmark_items( + kb_instance=NoQueryKnowledgeBase(), + db_id="db_1", + count=1, + neighbors_count=1, + llm_model_spec="test-model", + ) + ] + + assert items == [{"query": "问题", "gold_chunk_ids": ["anchor_chunk"], "gold_answer": "答案"}] + assert "片段ID=anchor_chunk" in fake_llm.prompts[0] + + +@pytest.mark.asyncio +async def test_select_neighbor_chunks_by_kb_query_filters_anchor(): + kb = FakeGenerationKnowledgeBase( + query_results=[ + { + "content": "anchor content", + "metadata": {"chunk_id": "anchor_chunk", "file_id": "file_a", "chunk_index": 0}, + }, + { + "content": "neighbor content", + "metadata": {"chunk_id": "neighbor_chunk", "file_id": "file_a", "chunk_index": 1}, + }, + ] + ) + + chunks = await select_neighbor_chunks_by_kb_query( + kb_instance=kb, + db_id="db_1", + anchor_chunk={"id": "anchor_chunk", "content": "anchor content", "file_id": "file_a", "chunk_index": 0}, + neighbors_count=1, + ) + + assert chunks == [{"id": "neighbor_chunk", "content": "neighbor content", "file_id": "file_a", "chunk_index": 1}] + assert kb.query_calls == [ + { + "query_text": "anchor content", + "db_id": "db_1", + "search_mode": "vector", + "final_top_k": 4, + "use_reranker": False, + "similarity_threshold": 0.0, + } + ] + + +@pytest.mark.asyncio +async def test_iter_generated_benchmark_items_uses_query_neighbor(monkeypatch): + fake_llm = FakeLlm(gold_chunk_id="neighbor_chunk") + monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm) + kb = FakeGenerationKnowledgeBase( + query_results=[ + { + "content": "neighbor content", + "metadata": {"chunk_id": "neighbor_chunk", "file_id": "file_a", "chunk_index": 1}, + } + ] + ) + + items = [ + item + async for item in iter_generated_benchmark_items( + kb_instance=kb, + db_id="db_1", + count=1, + neighbors_count=2, + llm_model_spec="test-model", + ) + ] + + assert items == [{"query": "问题", "gold_chunk_ids": ["neighbor_chunk"], "gold_answer": "答案"}] + assert kb.query_calls[0]["query_text"] == "anchor content" + assert kb.query_calls[0]["search_mode"] == "vector" + assert "片段ID=neighbor_chunk" in fake_llm.prompts[0] + + +@pytest.mark.asyncio +async def test_iter_generated_benchmark_items_falls_back_to_anchor_when_query_empty(monkeypatch): + fake_llm = FakeLlm() + monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm) + + items = [ + item + async for item in iter_generated_benchmark_items( + kb_instance=FakeGenerationKnowledgeBase(query_results=[]), + db_id="db_1", + count=1, + neighbors_count=2, + llm_model_spec="test-model", + ) + ] + + assert items == [{"query": "问题", "gold_chunk_ids": ["anchor_chunk"], "gold_answer": "答案"}] + assert "片段ID=anchor_chunk" in fake_llm.prompts[0] diff --git a/backend/test/unit/plugins/test_milvus_kb.py b/backend/test/unit/plugins/test_milvus_kb.py index d52ccc59..094cbe3c 100644 --- a/backend/test/unit/plugins/test_milvus_kb.py +++ b/backend/test/unit/plugins/test_milvus_kb.py @@ -1,6 +1,6 @@ from pymilvus import CollectionSchema, DataType, FieldSchema, Function, FunctionType -from yuxi.knowledge.implementations.milvus import CONTENT_ANALYZER_PARAMS, CONTENT_SPARSE_FIELD, MilvusKB +from yuxi.knowledge.implementations.milvus import CONTENT_ANALYZER_PARAMS, CONTENT_SPARSE_FIELD, MilvusKB, VECTOR_METRIC_TYPE class FakeHit: @@ -67,6 +67,18 @@ async def test_keyword_mode_uses_milvus_bm25_search(): assert search_call["limit"] == 7 +async def test_vector_mode_ignores_metric_type_override(): + collection = FakeCollection() + kb = make_kb(collection) + + chunks = await kb.aquery("vector query", "db", search_mode="vector", metric_type="L2") + + assert chunks[0]["content"] == "BM25 result" + search_call = collection.search_calls[0] + assert search_call["anns_field"] == "embedding" + assert search_call["param"]["metric_type"] == VECTOR_METRIC_TYPE + + async def test_hybrid_mode_uses_milvus_native_hybrid_search(): collection = FakeCollection() kb = make_kb(collection) @@ -90,6 +102,7 @@ async def test_hybrid_mode_uses_milvus_native_hybrid_search(): vector_request, bm25_request = hybrid_call["reqs"] assert vector_request.anns_field == "embedding" assert vector_request.data == [[0.1, 0.2]] + assert vector_request.param["metric_type"] == VECTOR_METRIC_TYPE assert bm25_request.anns_field == CONTENT_SPARSE_FIELD assert bm25_request.data == ["hybrid query"] assert bm25_request.limit == 8 @@ -118,6 +131,7 @@ def test_query_params_config_uses_bm25_parameters(): option_keys = {option["key"] for option in config["options"]} assert "keyword_top_k" not in option_keys + assert "metric_type" not in option_keys assert { "bm25_top_k", "vector_weight", diff --git a/docs/develop-guides/roadmap.md b/docs/develop-guides/roadmap.md index 10b7ab2e..9663d03d 100644 --- a/docs/develop-guides/roadmap.md +++ b/docs/develop-guides/roadmap.md @@ -23,7 +23,6 @@ ### Bugs - 目前的知识库的图片存在公开访问风险 -- 生成基准测试会把所有的向量都计算一遍不合理 ### BREAKING CHANGE（不兼容变更，0.7 版本再实现） - 将自定义provider 的实现逻辑，从文件移动到数据库中，并将相关处理代码，移出 config 文件，放到 provider 模块中 @@ -37,6 +36,7 @@ ### 0.6.2 开发记录 +- 优化评估基准自动生成：仅支持 commonrag/Milvus 知识库，默认参考 chunks 数量改为 1；多 chunk 场景复用知识库向量检索选择相似 chunks，不再对全量 chunks 重新计算 embedding，并移除前端 Embedding 模型选择。 - 修复知识库文档入库状态回退：当已解析文件缺失 `markdown_file` 解析产物时，索引流程会将文件状态恢复为未解析，便于重新解析而不是停留在索引失败。 - 优化 Agent 输入框文件 mention：用户级 workspace 文件候选改为从独立 workspace API 递归加载，不再依赖 active thread；插入时仍转换为 `/home/gem/user-data/workspace/` 沙盒虚拟路径，并修复附件上传后未立即刷新 mention 候选的问题。 - 调整知识库思维导图后端结构：将思维导图路由文件重命名为知识库语义更明确的 router，并把文件列表整理、提示词构建、AI JSON 解析等纯逻辑下沉到知识库 utils。 diff --git a/web/src/apis/knowledge_api.js b/web/src/apis/knowledge_api.js index 96ceca75..7ce2a712 100644 --- a/web/src/apis/knowledge_api.js +++ b/web/src/apis/knowledge_api.js @@ -479,8 +479,7 @@ export const evaluationApi = { * @param {string} dbId - 知识库ID * @param {Object} params - 生成参数 * @param {number} params.count - 生成问题数量 - * @param {boolean} params.include_answers - 是否生成答案 - * @param {Object} params.llm_config - LLM配置 + * @param {string} params.llm_model_spec - LLM模型配置 * @returns {Promise} - 生成结果 */ generateBenchmark: async (dbId, params) => { diff --git a/web/src/components/modals/BenchmarkGenerateModal.vue b/web/src/components/modals/BenchmarkGenerateModal.vue index 5fa6b216..3cdb7530 100644 --- a/web/src/components/modals/BenchmarkGenerateModal.vue +++ b/web/src/components/modals/BenchmarkGenerateModal.vue @@ -50,77 +50,23 @@ :min="0" :max="10" style="width: 100%" - placeholder="每次选取的相似chunks数量" + placeholder="每次参考的chunks总数" /> - - - - - - - - - - - - - - - - - - - - - - - + + @@ -130,8 +76,10 @@ import { ref, reactive, computed, watch, h } from 'vue' import { message } from 'ant-design-vue' import { evaluationApi } from '@/apis/knowledge_api' +import { useConfigStore } from '@/stores/config' import ModelSelectorComponent from '@/components/ModelSelectorComponent.vue' -import EmbeddingModelSelector from '@/components/EmbeddingModelSelector.vue' + +const configStore = useConfigStore() const props = defineProps({ visible: { @@ -146,22 +94,23 @@ const props = defineProps({ const emit = defineEmits(['update:visible', 'success']) +// 默认基准名称 +const defaultBenchmarkName = () => { + const today = new Date().toISOString().slice(0, 10) + const suffix = Array.from({ length: 4 }, () => '0123456789abcdefghijklmnopqrstuvwxyz'[Math.floor(Math.random() * 36)]).join('') + return `Test-${today}-${suffix}` +} + // 响应式数据 const formRef = ref() const generating = ref(false) const formState = reactive({ - name: '', + name: defaultBenchmarkName(), description: '', count: 10, - neighbors_count: 2, - embedding_model_id: '', - llm_model_spec: '', - llm_config: { - model: '', - temperature: 0.7, - max_tokens: 1000 - } + neighbors_count: 1, + llm_model_spec: configStore.config?.default_model || '' }) // 表单验证规则 @@ -208,11 +157,7 @@ const handleGenerate = async () => { description: formState.description, count: formState.count, neighbors_count: formState.neighbors_count, - embedding_model_id: formState.embedding_model_id, - llm_config: { - ...formState.llm_config, - model_spec: formState.llm_model_spec - } + llm_model_spec: formState.llm_model_spec } const response = await evaluationApi.generateBenchmark(props.databaseId, params) @@ -242,17 +187,11 @@ const handleCancel = () => { const resetForm = () => { formRef.value?.resetFields() Object.assign(formState, { - name: '', + name: defaultBenchmarkName(), description: '', count: 10, - neighbors_count: 2, - embedding_model_id: '', - llm_model_spec: '', - llm_config: { - model: '', - temperature: 0.7, - max_tokens: 1000 - } + neighbors_count: 1, + llm_model_spec: configStore.config?.default_model || '' }) generating.value = false } @@ -260,28 +199,14 @@ const resetForm = () => { // 选择LLM模型 const handleSelectLLMModel = (modelSpec) => { formState.llm_model_spec = modelSpec - formState.llm_config.model = modelSpec } // 监听visible变化 watch(visible, (val) => { - if (!val) { + if (val) { resetForm() } }) -