feat: 优化评估基准生成，支持仅使用 commonrag/Milvus 知识库，调整参数配置
This commit is contained in:
Wenjie Zhang
parent
179b048f07
commit
ab513e2619
@ -1,13 +1,11 @@
|
||||
import json
|
||||
import math
|
||||
import random
|
||||
from collections.abc import AsyncIterator, Callable
|
||||
from typing import Any
|
||||
|
||||
import json_repair
|
||||
|
||||
from yuxi import config
|
||||
from yuxi.models import select_embedding_model, select_model
|
||||
from yuxi.models import select_model
|
||||
from yuxi.utils import logger
|
||||
|
||||
|
||||
@ -36,26 +34,59 @@ def clamp_neighbors_count(neighbors_count: int) -> int:
|
||||
return min(max(neighbors_count, 0), 10)
|
||||
|
||||
|
||||
def cosine_similarity(a: list[float], b: list[float], na: float, nb: float) -> float:
|
||||
s = 0.0
|
||||
for i in range(len(a)):
|
||||
s += a[i] * b[i]
|
||||
return s / (na * nb)
|
||||
def _is_anchor_chunk(candidate: dict[str, Any], anchor_chunk: dict[str, Any]) -> bool:
|
||||
metadata = candidate.get("metadata") or {}
|
||||
candidate_id = metadata.get("chunk_id")
|
||||
if candidate_id is not None and str(candidate_id) == str(anchor_chunk.get("id")):
|
||||
return True
|
||||
|
||||
candidate_file_id = metadata.get("file_id")
|
||||
candidate_chunk_index = metadata.get("chunk_index")
|
||||
return candidate_file_id == anchor_chunk.get("file_id") and candidate_chunk_index == anchor_chunk.get("chunk_index")
|
||||
|
||||
|
||||
def select_neighbor_indices(
|
||||
anchor_idx: int, embeddings: list[list[float]], norms: list[float], neighbors_count: int
|
||||
) -> list[int]:
|
||||
anchor_embedding = embeddings[anchor_idx]
|
||||
anchor_norm = norms[anchor_idx]
|
||||
sims = []
|
||||
for idx in range(len(embeddings)):
|
||||
if idx == anchor_idx:
|
||||
async def select_neighbor_chunks_by_kb_query(
|
||||
*, kb_instance: Any, db_id: str, anchor_chunk: dict[str, Any], neighbors_count: int
|
||||
) -> list[dict[str, Any]]:
|
||||
if neighbors_count <= 0:
|
||||
return []
|
||||
|
||||
anchor_content = anchor_chunk.get("content", "")
|
||||
if not anchor_content:
|
||||
return []
|
||||
|
||||
candidates = await kb_instance.aquery(
|
||||
anchor_content,
|
||||
db_id,
|
||||
search_mode="vector",
|
||||
final_top_k=neighbors_count + 3,
|
||||
use_reranker=False,
|
||||
similarity_threshold=0.0,
|
||||
)
|
||||
|
||||
chunks = []
|
||||
for candidate in candidates:
|
||||
if _is_anchor_chunk(candidate, anchor_chunk):
|
||||
continue
|
||||
score = cosine_similarity(anchor_embedding, embeddings[idx], anchor_norm, norms[idx])
|
||||
sims.append((idx, score))
|
||||
sims.sort(key=lambda x: x[1], reverse=True)
|
||||
return [idx for idx, _ in sims[:neighbors_count]]
|
||||
|
||||
metadata = candidate.get("metadata") or {}
|
||||
chunk_id = metadata.get("chunk_id")
|
||||
content = candidate.get("content", "")
|
||||
if not chunk_id or not content:
|
||||
continue
|
||||
|
||||
chunks.append(
|
||||
{
|
||||
"id": str(chunk_id),
|
||||
"content": content,
|
||||
"file_id": metadata.get("file_id"),
|
||||
"chunk_index": metadata.get("chunk_index"),
|
||||
}
|
||||
)
|
||||
if len(chunks) >= neighbors_count:
|
||||
break
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
def build_benchmark_generation_prompt(ctx_items: list[tuple[str, str]]) -> str:
|
||||
@ -74,7 +105,6 @@ async def iter_generated_benchmark_items(
|
||||
db_id: str,
|
||||
count: int,
|
||||
neighbors_count: int,
|
||||
embedding_model_id: str,
|
||||
llm_model_spec: Any,
|
||||
progress_cb: Callable[[int, str], Any] | None = None,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
@ -86,31 +116,25 @@ async def iter_generated_benchmark_items(
|
||||
raise ValueError("No chunks found in knowledge base")
|
||||
|
||||
if progress_cb:
|
||||
await progress_cb(15, "向量化")
|
||||
await progress_cb(15, "准备生成样本")
|
||||
|
||||
contents = [chunk["content"] for chunk in all_chunks]
|
||||
embed_model = select_embedding_model(embedding_model_id)
|
||||
batch_size = int(getattr(embed_model, "batch_size", 40) or 40)
|
||||
if embedding_model_id in config.embed_model_names:
|
||||
batch_size = config.embed_model_names[embedding_model_id].batch_size
|
||||
|
||||
embeddings = await embed_model.abatch_encode(contents, batch_size=batch_size)
|
||||
norms = [math.sqrt(sum(x * x for x in vec)) or 1.0 for vec in embeddings]
|
||||
llm = select_model(model_spec=llm_model_spec)
|
||||
neighbors_count = clamp_neighbors_count(neighbors_count)
|
||||
context_count = max(clamp_neighbors_count(neighbors_count), 1)
|
||||
generated = 0
|
||||
attempts = 0
|
||||
max_attempts = max(count * 5, 50)
|
||||
|
||||
if progress_cb:
|
||||
await progress_cb(0, "准备生成样本")
|
||||
|
||||
while generated < count and attempts < max_attempts:
|
||||
attempts += 1
|
||||
anchor_idx = random.randrange(len(all_chunks))
|
||||
neighbor_indices = select_neighbor_indices(anchor_idx, embeddings, norms, neighbors_count)
|
||||
ctx_items = [(all_chunks[anchor_idx]["id"], all_chunks[anchor_idx]["content"])]
|
||||
ctx_items.extend((all_chunks[idx]["id"], all_chunks[idx]["content"]) for idx in neighbor_indices)
|
||||
anchor_chunk = all_chunks[random.randrange(len(all_chunks))]
|
||||
neighbor_chunks = await select_neighbor_chunks_by_kb_query(
|
||||
kb_instance=kb_instance,
|
||||
db_id=db_id,
|
||||
anchor_chunk=anchor_chunk,
|
||||
neighbors_count=context_count - 1,
|
||||
)
|
||||
ctx_chunks = [anchor_chunk] + neighbor_chunks
|
||||
ctx_items = [(chunk["id"], chunk["content"]) for chunk in ctx_chunks]
|
||||
allowed_ids = {cid for cid, _ in ctx_items}
|
||||
|
||||
try:
|
||||
|
||||
@ -32,6 +32,7 @@ from yuxi.utils.datetime_utils import utc_isoformat
|
||||
MILVUS_AVAILABLE = True
|
||||
CONTENT_SPARSE_FIELD = "content_sparse"
|
||||
CONTENT_ANALYZER_PARAMS = {"type": "chinese"}
|
||||
VECTOR_METRIC_TYPE = "COSINE"
|
||||
|
||||
|
||||
class MilvusKB(KnowledgeBase):
|
||||
@ -189,7 +190,7 @@ class MilvusKB(KnowledgeBase):
|
||||
collection = Collection(name=collection_name, schema=schema, using=self.connection_alias)
|
||||
|
||||
# 创建索引
|
||||
index_params = {"metric_type": "COSINE", "index_type": "IVF_FLAT", "params": {"nlist": 1024}}
|
||||
index_params = {"metric_type": VECTOR_METRIC_TYPE, "index_type": "IVF_FLAT", "params": {"nlist": 1024}}
|
||||
collection.create_index("embedding", index_params)
|
||||
sparse_index_params = {
|
||||
"metric_type": "BM25",
|
||||
@ -572,7 +573,7 @@ class MilvusKB(KnowledgeBase):
|
||||
final_top_k = int(merged_kwargs.get("final_top_k", 10))
|
||||
final_top_k = max(final_top_k, 1)
|
||||
similarity_threshold = float(merged_kwargs.get("similarity_threshold", 0.2))
|
||||
metric_type = merged_kwargs.get("metric_type", "COSINE")
|
||||
metric_type = VECTOR_METRIC_TYPE
|
||||
include_distances = bool(merged_kwargs.get("include_distances", True))
|
||||
search_mode = str(merged_kwargs.get("search_mode", "vector")).lower()
|
||||
if search_mode not in {"vector", "keyword", "hybrid"}:
|
||||
@ -615,7 +616,7 @@ class MilvusKB(KnowledgeBase):
|
||||
|
||||
if results and len(results) > 0 and len(results[0]) > 0:
|
||||
for hit in results[0]:
|
||||
similarity = hit.distance if metric_type == "COSINE" else 1 / (1 + hit.distance)
|
||||
similarity = hit.distance if metric_type == VECTOR_METRIC_TYPE else 1 / (1 + hit.distance)
|
||||
if similarity < similarity_threshold:
|
||||
continue
|
||||
|
||||
@ -938,18 +939,6 @@ class MilvusKB(KnowledgeBase):
|
||||
"default": True,
|
||||
"description": "在结果中显示相似度分数",
|
||||
},
|
||||
{
|
||||
"key": "metric_type",
|
||||
"label": "距离度量类型",
|
||||
"type": "select",
|
||||
"default": "COSINE",
|
||||
"options": [
|
||||
{"value": "COSINE", "label": "余弦相似度", "description": "适合文本语义相似度"},
|
||||
{"value": "L2", "label": "欧几里得距离", "description": "适合数值型数据"},
|
||||
{"value": "IP", "label": "内积", "description": "适合标准化向量"},
|
||||
],
|
||||
"description": "向量相似度计算方法",
|
||||
},
|
||||
{
|
||||
"key": "use_reranker",
|
||||
"label": "启用重排序",
|
||||
|
||||
@ -297,25 +297,17 @@ class EvaluationService:
|
||||
name = payload.get("name", "自动生成评估基准")
|
||||
description = payload.get("description", "")
|
||||
count = int(payload.get("count", 10))
|
||||
neighbors_count = int(payload.get("neighbors_count", 0))
|
||||
embedding_model_id = payload.get("embedding_model_id")
|
||||
llm_model_spec = payload.get("llm_model_spec") or (payload.get("llm_config") or {}).get("model_spec")
|
||||
neighbors_count = int(payload.get("neighbors_count", 1))
|
||||
llm_model_spec = payload.get("llm_model_spec")
|
||||
|
||||
kb_instance = await knowledge_base.aget_kb(db_id)
|
||||
if not kb_instance:
|
||||
await context.set_message("知识库不存在")
|
||||
raise ValueError("Knowledge Base not found")
|
||||
if kb_instance.kb_type == "lightrag":
|
||||
await context.set_message("暂不支持该类型知识库生成评估基准")
|
||||
if kb_instance.kb_type != "milvus":
|
||||
await context.set_message("仅支持 commonrag/Milvus 类型知识库生成评估基准")
|
||||
raise ValueError("Unsupported KB type for benchmark generation")
|
||||
|
||||
db_meta = kb_instance.databases_meta.get(db_id, {})
|
||||
embed_info = db_meta.get("embed_info", {})
|
||||
if not embedding_model_id:
|
||||
embedding_model_id = embed_info.get("name") or embed_info.get("model") or ""
|
||||
if not embedding_model_id:
|
||||
raise ValueError("Embedding model not specified")
|
||||
|
||||
benchmark_id = f"benchmark_{uuid.uuid4().hex[:8]}"
|
||||
bench_dir = await self._get_benchmark_dir(db_id)
|
||||
data_file_path = os.path.join(bench_dir, f"{benchmark_id}.jsonl")
|
||||
@ -328,7 +320,6 @@ class EvaluationService:
|
||||
db_id=db_id,
|
||||
count=count,
|
||||
neighbors_count=neighbors_count,
|
||||
embedding_model_id=embedding_model_id,
|
||||
llm_model_spec=llm_model_spec,
|
||||
progress_cb=context.set_progress,
|
||||
):
|
||||
|
||||
@ -1,15 +1,17 @@
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
os.environ.setdefault("OPENAI_API_KEY", "test-key")
|
||||
|
||||
from yuxi.knowledge.eval import benchmark_generation
|
||||
from yuxi.knowledge.eval.benchmark_generation import (
|
||||
build_benchmark_generation_prompt,
|
||||
clamp_neighbors_count,
|
||||
collect_kb_chunks,
|
||||
cosine_similarity,
|
||||
select_neighbor_indices,
|
||||
iter_generated_benchmark_items,
|
||||
select_neighbor_chunks_by_kb_query,
|
||||
)
|
||||
|
||||
|
||||
@ -27,19 +29,48 @@ class FakeKnowledgeBase:
|
||||
}
|
||||
|
||||
|
||||
class FakeGenerationKnowledgeBase:
|
||||
files_meta = {"file_a": {"database_id": "db_1"}}
|
||||
|
||||
def __init__(self, query_results=None):
|
||||
self.query_results = query_results or []
|
||||
self.query_calls = []
|
||||
|
||||
async def get_file_content(self, db_id, fid):
|
||||
return {
|
||||
"lines": [
|
||||
{"id": "anchor_chunk", "content": "anchor content", "chunk_order_index": 0},
|
||||
]
|
||||
}
|
||||
|
||||
async def aquery(self, query_text, db_id, **kwargs):
|
||||
self.query_calls.append({"query_text": query_text, "db_id": db_id, **kwargs})
|
||||
return self.query_results
|
||||
|
||||
|
||||
class FakeLlm:
|
||||
def __init__(self, gold_chunk_id="anchor_chunk"):
|
||||
self.gold_chunk_id = gold_chunk_id
|
||||
self.prompts = []
|
||||
|
||||
async def call(self, prompt, stream):
|
||||
self.prompts.append(prompt)
|
||||
return SimpleNamespace(
|
||||
content=('{"query":"问题","gold_answer":"答案","gold_chunk_ids":["' + self.gold_chunk_id + '"]}')
|
||||
)
|
||||
|
||||
|
||||
class NoQueryKnowledgeBase(FakeGenerationKnowledgeBase):
|
||||
async def aquery(self, query_text, db_id, **kwargs):
|
||||
raise AssertionError("neighbors_count=1 时不应调用 aquery")
|
||||
|
||||
|
||||
def test_clamp_neighbors_count():
|
||||
assert clamp_neighbors_count(-1) == 0
|
||||
assert clamp_neighbors_count(3) == 3
|
||||
assert clamp_neighbors_count(11) == 10
|
||||
|
||||
|
||||
def test_select_neighbor_indices_orders_by_cosine_similarity():
|
||||
embeddings = [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]
|
||||
norms = [1.0, cosine_similarity(embeddings[1], embeddings[1], 1.0, 1.0) ** 0.5, 1.0]
|
||||
|
||||
assert select_neighbor_indices(0, embeddings, norms, 1) == [1]
|
||||
|
||||
|
||||
def test_build_benchmark_generation_prompt_contains_required_schema():
|
||||
prompt = build_benchmark_generation_prompt([("chunk_1", "片段内容")])
|
||||
|
||||
@ -52,3 +83,108 @@ async def test_collect_kb_chunks_filters_database_id():
|
||||
chunks = await collect_kb_chunks(FakeKnowledgeBase(), "db_1")
|
||||
|
||||
assert chunks == [{"id": "file_a_chunk", "content": "内容", "file_id": "file_a", "chunk_index": 0}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_iter_generated_benchmark_items_with_one_chunk_does_not_query(monkeypatch):
|
||||
fake_llm = FakeLlm()
|
||||
monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
|
||||
|
||||
items = [
|
||||
item
|
||||
async for item in iter_generated_benchmark_items(
|
||||
kb_instance=NoQueryKnowledgeBase(),
|
||||
db_id="db_1",
|
||||
count=1,
|
||||
neighbors_count=1,
|
||||
llm_model_spec="test-model",
|
||||
)
|
||||
]
|
||||
|
||||
assert items == [{"query": "问题", "gold_chunk_ids": ["anchor_chunk"], "gold_answer": "答案"}]
|
||||
assert "片段ID=anchor_chunk" in fake_llm.prompts[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_select_neighbor_chunks_by_kb_query_filters_anchor():
|
||||
kb = FakeGenerationKnowledgeBase(
|
||||
query_results=[
|
||||
{
|
||||
"content": "anchor content",
|
||||
"metadata": {"chunk_id": "anchor_chunk", "file_id": "file_a", "chunk_index": 0},
|
||||
},
|
||||
{
|
||||
"content": "neighbor content",
|
||||
"metadata": {"chunk_id": "neighbor_chunk", "file_id": "file_a", "chunk_index": 1},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
chunks = await select_neighbor_chunks_by_kb_query(
|
||||
kb_instance=kb,
|
||||
db_id="db_1",
|
||||
anchor_chunk={"id": "anchor_chunk", "content": "anchor content", "file_id": "file_a", "chunk_index": 0},
|
||||
neighbors_count=1,
|
||||
)
|
||||
|
||||
assert chunks == [{"id": "neighbor_chunk", "content": "neighbor content", "file_id": "file_a", "chunk_index": 1}]
|
||||
assert kb.query_calls == [
|
||||
{
|
||||
"query_text": "anchor content",
|
||||
"db_id": "db_1",
|
||||
"search_mode": "vector",
|
||||
"final_top_k": 4,
|
||||
"use_reranker": False,
|
||||
"similarity_threshold": 0.0,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_iter_generated_benchmark_items_uses_query_neighbor(monkeypatch):
|
||||
fake_llm = FakeLlm(gold_chunk_id="neighbor_chunk")
|
||||
monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
|
||||
kb = FakeGenerationKnowledgeBase(
|
||||
query_results=[
|
||||
{
|
||||
"content": "neighbor content",
|
||||
"metadata": {"chunk_id": "neighbor_chunk", "file_id": "file_a", "chunk_index": 1},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
items = [
|
||||
item
|
||||
async for item in iter_generated_benchmark_items(
|
||||
kb_instance=kb,
|
||||
db_id="db_1",
|
||||
count=1,
|
||||
neighbors_count=2,
|
||||
llm_model_spec="test-model",
|
||||
)
|
||||
]
|
||||
|
||||
assert items == [{"query": "问题", "gold_chunk_ids": ["neighbor_chunk"], "gold_answer": "答案"}]
|
||||
assert kb.query_calls[0]["query_text"] == "anchor content"
|
||||
assert kb.query_calls[0]["search_mode"] == "vector"
|
||||
assert "片段ID=neighbor_chunk" in fake_llm.prompts[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_iter_generated_benchmark_items_falls_back_to_anchor_when_query_empty(monkeypatch):
|
||||
fake_llm = FakeLlm()
|
||||
monkeypatch.setattr(benchmark_generation, "select_model", lambda model_spec: fake_llm)
|
||||
|
||||
items = [
|
||||
item
|
||||
async for item in iter_generated_benchmark_items(
|
||||
kb_instance=FakeGenerationKnowledgeBase(query_results=[]),
|
||||
db_id="db_1",
|
||||
count=1,
|
||||
neighbors_count=2,
|
||||
llm_model_spec="test-model",
|
||||
)
|
||||
]
|
||||
|
||||
assert items == [{"query": "问题", "gold_chunk_ids": ["anchor_chunk"], "gold_answer": "答案"}]
|
||||
assert "片段ID=anchor_chunk" in fake_llm.prompts[0]
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
from pymilvus import CollectionSchema, DataType, FieldSchema, Function, FunctionType
|
||||
|
||||
from yuxi.knowledge.implementations.milvus import CONTENT_ANALYZER_PARAMS, CONTENT_SPARSE_FIELD, MilvusKB
|
||||
from yuxi.knowledge.implementations.milvus import CONTENT_ANALYZER_PARAMS, CONTENT_SPARSE_FIELD, MilvusKB, VECTOR_METRIC_TYPE
|
||||
|
||||
|
||||
class FakeHit:
|
||||
@ -67,6 +67,18 @@ async def test_keyword_mode_uses_milvus_bm25_search():
|
||||
assert search_call["limit"] == 7
|
||||
|
||||
|
||||
async def test_vector_mode_ignores_metric_type_override():
|
||||
collection = FakeCollection()
|
||||
kb = make_kb(collection)
|
||||
|
||||
chunks = await kb.aquery("vector query", "db", search_mode="vector", metric_type="L2")
|
||||
|
||||
assert chunks[0]["content"] == "BM25 result"
|
||||
search_call = collection.search_calls[0]
|
||||
assert search_call["anns_field"] == "embedding"
|
||||
assert search_call["param"]["metric_type"] == VECTOR_METRIC_TYPE
|
||||
|
||||
|
||||
async def test_hybrid_mode_uses_milvus_native_hybrid_search():
|
||||
collection = FakeCollection()
|
||||
kb = make_kb(collection)
|
||||
@ -90,6 +102,7 @@ async def test_hybrid_mode_uses_milvus_native_hybrid_search():
|
||||
vector_request, bm25_request = hybrid_call["reqs"]
|
||||
assert vector_request.anns_field == "embedding"
|
||||
assert vector_request.data == [[0.1, 0.2]]
|
||||
assert vector_request.param["metric_type"] == VECTOR_METRIC_TYPE
|
||||
assert bm25_request.anns_field == CONTENT_SPARSE_FIELD
|
||||
assert bm25_request.data == ["hybrid query"]
|
||||
assert bm25_request.limit == 8
|
||||
@ -118,6 +131,7 @@ def test_query_params_config_uses_bm25_parameters():
|
||||
|
||||
option_keys = {option["key"] for option in config["options"]}
|
||||
assert "keyword_top_k" not in option_keys
|
||||
assert "metric_type" not in option_keys
|
||||
assert {
|
||||
"bm25_top_k",
|
||||
"vector_weight",
|
||||
|
||||
@ -23,7 +23,6 @@
|
||||
|
||||
### Bugs
|
||||
- 目前的知识库的图片存在公开访问风险
|
||||
- 生成基准测试会把所有的向量都计算一遍不合理
|
||||
|
||||
### BREAKING CHANGE（不兼容变更，0.7 版本再实现）
|
||||
- 将自定义provider 的实现逻辑，从文件移动到数据库中，并将相关处理代码，移出 config 文件，放到 provider 模块中
|
||||
@ -37,6 +36,7 @@
|
||||
### 0.6.2 开发记录
|
||||
|
||||
<!-- 0.6.2 的内容请放在这里 -->
|
||||
- 优化评估基准自动生成：仅支持 commonrag/Milvus 知识库，默认参考 chunks 数量改为 1；多 chunk 场景复用知识库向量检索选择相似 chunks，不再对全量 chunks 重新计算 embedding，并移除前端 Embedding 模型选择。
|
||||
- 修复知识库文档入库状态回退：当已解析文件缺失 `markdown_file` 解析产物时，索引流程会将文件状态恢复为未解析，便于重新解析而不是停留在索引失败。
|
||||
- 优化 Agent 输入框文件 mention：用户级 workspace 文件候选改为从独立 workspace API 递归加载，不再依赖 active thread；插入时仍转换为 `/home/gem/user-data/workspace/` 沙盒虚拟路径，并修复附件上传后未立即刷新 mention 候选的问题。
|
||||
- 调整知识库思维导图后端结构：将思维导图路由文件重命名为知识库语义更明确的 router，并把文件列表整理、提示词构建、AI JSON 解析等纯逻辑下沉到知识库 utils。
|
||||
|
||||
@ -479,8 +479,7 @@ export const evaluationApi = {
|
||||
* @param {string} dbId - 知识库ID
|
||||
* @param {Object} params - 生成参数
|
||||
* @param {number} params.count - 生成问题数量
|
||||
* @param {boolean} params.include_answers - 是否生成答案
|
||||
* @param {Object} params.llm_config - LLM配置
|
||||
* @param {string} params.llm_model_spec - LLM模型配置
|
||||
* @returns {Promise} - 生成结果
|
||||
*/
|
||||
generateBenchmark: async (dbId, params) => {
|
||||
|
||||
@ -50,77 +50,23 @@
|
||||
:min="0"
|
||||
:max="10"
|
||||
style="width: 100%"
|
||||
placeholder="每次选取的相似chunks数量"
|
||||
placeholder="每次参考的chunks总数"
|
||||
/>
|
||||
</a-form-item>
|
||||
</a-col>
|
||||
</a-row>
|
||||
</a-form-item>
|
||||
|
||||
<a-form-item label="LLM配置" name="llm_config">
|
||||
<a-card size="small" title="配置参数">
|
||||
<a-form-item
|
||||
label="LLM模型配置"
|
||||
name="llm_model_spec"
|
||||
:rules="[{ required: true, message: '请选择LLM模型' }]"
|
||||
>
|
||||
<ModelSelectorComponent
|
||||
:model_spec="formState.llm_model_spec"
|
||||
placeholder="选择用于生成问题的LLM模型"
|
||||
size="small"
|
||||
@select-model="handleSelectLLMModel"
|
||||
/>
|
||||
</a-form-item>
|
||||
|
||||
<a-form-item
|
||||
label="Embedding模型"
|
||||
name="embedding_model_id"
|
||||
:rules="[{ required: true, message: '请选择Embedding模型' }]"
|
||||
>
|
||||
<EmbeddingModelSelector
|
||||
v-model:value="formState.embedding_model_id"
|
||||
placeholder="请选择用于相似度计算的Embedding模型"
|
||||
size="default"
|
||||
/>
|
||||
</a-form-item>
|
||||
|
||||
<a-row :gutter="16">
|
||||
<a-col :span="12">
|
||||
<a-form-item
|
||||
label="Temperature"
|
||||
name="temperature"
|
||||
:labelCol="{ span: 24 }"
|
||||
:wrapperCol="{ span: 24 }"
|
||||
>
|
||||
<a-input-number
|
||||
v-model:value="formState.llm_config.temperature"
|
||||
:min="0"
|
||||
:max="2"
|
||||
:step="0.1"
|
||||
style="width: 100%"
|
||||
placeholder="控制生成内容的随机性"
|
||||
/>
|
||||
</a-form-item>
|
||||
</a-col>
|
||||
<a-col :span="12">
|
||||
<a-form-item
|
||||
label="Max Tokens"
|
||||
name="max_tokens"
|
||||
:labelCol="{ span: 24 }"
|
||||
:wrapperCol="{ span: 24 }"
|
||||
>
|
||||
<a-input-number
|
||||
v-model:value="formState.llm_config.max_tokens"
|
||||
:min="100"
|
||||
:max="4000"
|
||||
:step="100"
|
||||
style="width: 100%"
|
||||
placeholder="生成内容的最大长度"
|
||||
/>
|
||||
</a-form-item>
|
||||
</a-col>
|
||||
</a-row>
|
||||
</a-card>
|
||||
<a-form-item
|
||||
label="LLM模型配置"
|
||||
name="llm_model_spec"
|
||||
:rules="[{ required: true, message: '请选择LLM模型' }]"
|
||||
>
|
||||
<ModelSelectorComponent
|
||||
:model_spec="formState.llm_model_spec"
|
||||
placeholder="选择用于生成问题的LLM模型"
|
||||
@select-model="handleSelectLLMModel"
|
||||
/>
|
||||
</a-form-item>
|
||||
</a-form>
|
||||
</a-modal>
|
||||
@ -130,8 +76,10 @@
|
||||
import { ref, reactive, computed, watch, h } from 'vue'
|
||||
import { message } from 'ant-design-vue'
|
||||
import { evaluationApi } from '@/apis/knowledge_api'
|
||||
import { useConfigStore } from '@/stores/config'
|
||||
import ModelSelectorComponent from '@/components/ModelSelectorComponent.vue'
|
||||
import EmbeddingModelSelector from '@/components/EmbeddingModelSelector.vue'
|
||||
|
||||
const configStore = useConfigStore()
|
||||
|
||||
const props = defineProps({
|
||||
visible: {
|
||||
@ -146,22 +94,23 @@ const props = defineProps({
|
||||
|
||||
const emit = defineEmits(['update:visible', 'success'])
|
||||
|
||||
// 默认基准名称
|
||||
const defaultBenchmarkName = () => {
|
||||
const today = new Date().toISOString().slice(0, 10)
|
||||
const suffix = Array.from({ length: 4 }, () => '0123456789abcdefghijklmnopqrstuvwxyz'[Math.floor(Math.random() * 36)]).join('')
|
||||
return `Test-${today}-${suffix}`
|
||||
}
|
||||
|
||||
// 响应式数据
|
||||
const formRef = ref()
|
||||
const generating = ref(false)
|
||||
|
||||
const formState = reactive({
|
||||
name: '',
|
||||
name: defaultBenchmarkName(),
|
||||
description: '',
|
||||
count: 10,
|
||||
neighbors_count: 2,
|
||||
embedding_model_id: '',
|
||||
llm_model_spec: '',
|
||||
llm_config: {
|
||||
model: '',
|
||||
temperature: 0.7,
|
||||
max_tokens: 1000
|
||||
}
|
||||
neighbors_count: 1,
|
||||
llm_model_spec: configStore.config?.default_model || ''
|
||||
})
|
||||
|
||||
// 表单验证规则
|
||||
@ -208,11 +157,7 @@ const handleGenerate = async () => {
|
||||
description: formState.description,
|
||||
count: formState.count,
|
||||
neighbors_count: formState.neighbors_count,
|
||||
embedding_model_id: formState.embedding_model_id,
|
||||
llm_config: {
|
||||
...formState.llm_config,
|
||||
model_spec: formState.llm_model_spec
|
||||
}
|
||||
llm_model_spec: formState.llm_model_spec
|
||||
}
|
||||
|
||||
const response = await evaluationApi.generateBenchmark(props.databaseId, params)
|
||||
@ -242,17 +187,11 @@ const handleCancel = () => {
|
||||
const resetForm = () => {
|
||||
formRef.value?.resetFields()
|
||||
Object.assign(formState, {
|
||||
name: '',
|
||||
name: defaultBenchmarkName(),
|
||||
description: '',
|
||||
count: 10,
|
||||
neighbors_count: 2,
|
||||
embedding_model_id: '',
|
||||
llm_model_spec: '',
|
||||
llm_config: {
|
||||
model: '',
|
||||
temperature: 0.7,
|
||||
max_tokens: 1000
|
||||
}
|
||||
neighbors_count: 1,
|
||||
llm_model_spec: configStore.config?.default_model || ''
|
||||
})
|
||||
generating.value = false
|
||||
}
|
||||
@ -260,28 +199,14 @@ const resetForm = () => {
|
||||
// 选择LLM模型
|
||||
const handleSelectLLMModel = (modelSpec) => {
|
||||
formState.llm_model_spec = modelSpec
|
||||
formState.llm_config.model = modelSpec
|
||||
}
|
||||
|
||||
// 监听visible变化
|
||||
watch(visible, (val) => {
|
||||
if (!val) {
|
||||
if (val) {
|
||||
resetForm()
|
||||
}
|
||||
})
|
||||
</script>
|
||||
|
||||
<style lang="less" scoped>
|
||||
:deep(.ant-card) {
|
||||
.ant-card-head {
|
||||
min-height: auto;
|
||||
padding: 0 12px;
|
||||
border-bottom: 1px solid var(--gray-200);
|
||||
|
||||
.ant-card-head-title {
|
||||
font-size: 14px;
|
||||
padding: 8px 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user