refactor: 评估系统重构 - 后端统一 dataset/run 语义
- 评估数据集/题目/运行/逐题结果全量入库，JSONL 仅作为交换格式 - benchmark -> dataset, task -> run 术语统一 - 评估生成支持配置并发数 - 移除 base.py 中旧的 benchmarks_meta 元数据逻辑 - manager.py 统一表创建方法名
This commit is contained in:
Wenjie Zhang
parent
37179e233f
commit
1efbf86bf0
@ -1428,13 +1428,11 @@ class KnowledgeBase(ABC):
|
||||
return retrievers
|
||||
|
||||
async def _load_metadata(self) -> None:
|
||||
from yuxi.repositories.evaluation_repository import EvaluationRepository
|
||||
from yuxi.repositories.knowledge_base_repository import KnowledgeBaseRepository
|
||||
from yuxi.repositories.knowledge_file_repository import KnowledgeFileRepository
|
||||
|
||||
kb_repo = KnowledgeBaseRepository()
|
||||
file_repo = KnowledgeFileRepository()
|
||||
eval_repo = EvaluationRepository()
|
||||
|
||||
databases = [kb for kb in await kb_repo.get_all() if kb.kb_type == self.kb_type]
|
||||
self.databases_meta = {
|
||||
@ -1484,26 +1482,6 @@ class KnowledgeBase(ABC):
|
||||
}
|
||||
|
||||
self.benchmarks_meta = {}
|
||||
for kb in databases:
|
||||
benchmarks = await eval_repo.list_benchmarks(kb.db_id)
|
||||
if not benchmarks:
|
||||
continue
|
||||
self.benchmarks_meta[kb.db_id] = {}
|
||||
for bench in benchmarks:
|
||||
self.benchmarks_meta[kb.db_id][bench.benchmark_id] = {
|
||||
"id": bench.benchmark_id,
|
||||
"benchmark_id": bench.benchmark_id,
|
||||
"name": bench.name,
|
||||
"description": bench.description,
|
||||
"db_id": bench.db_id,
|
||||
"question_count": bench.question_count,
|
||||
"has_gold_chunks": bench.has_gold_chunks,
|
||||
"has_gold_answers": bench.has_gold_answers,
|
||||
"benchmark_file": bench.data_file_path,
|
||||
"created_by": bench.created_by,
|
||||
"created_at": utc_isoformat(bench.created_at) if bench.created_at else None,
|
||||
"updated_at": utc_isoformat(bench.updated_at) if bench.updated_at else None,
|
||||
}
|
||||
|
||||
logger.info(f"Loaded {self.kb_type} metadata from database for {len(self.databases_meta)} databases")
|
||||
await self._fill_missing_file_sizes()
|
||||
@ -1556,13 +1534,11 @@ class KnowledgeBase(ABC):
|
||||
await self._persist_file(file_id)
|
||||
|
||||
async def _save_metadata(self) -> None:
|
||||
from yuxi.repositories.evaluation_repository import EvaluationRepository
|
||||
from yuxi.repositories.knowledge_base_repository import KnowledgeBaseRepository
|
||||
from yuxi.repositories.knowledge_file_repository import KnowledgeFileRepository
|
||||
|
||||
kb_repo = KnowledgeBaseRepository()
|
||||
file_repo = KnowledgeFileRepository()
|
||||
eval_repo = EvaluationRepository()
|
||||
|
||||
self._normalize_metadata_state()
|
||||
|
||||
@ -1608,23 +1584,6 @@ class KnowledgeBase(ABC):
|
||||
},
|
||||
)
|
||||
|
||||
for db_id, benchmarks in self.benchmarks_meta.items():
|
||||
for benchmark_id, meta in benchmarks.items():
|
||||
existing = await eval_repo.get_benchmark(benchmark_id)
|
||||
payload = {
|
||||
"benchmark_id": benchmark_id,
|
||||
"db_id": db_id,
|
||||
"name": meta.get("name") or benchmark_id,
|
||||
"description": meta.get("description"),
|
||||
"question_count": int(meta.get("question_count") or 0),
|
||||
"has_gold_chunks": bool(meta.get("has_gold_chunks")),
|
||||
"has_gold_answers": bool(meta.get("has_gold_answers")),
|
||||
"data_file_path": meta.get("benchmark_file"),
|
||||
"created_by": str(meta.get("created_by")) if meta.get("created_by") else None,
|
||||
}
|
||||
if existing is None:
|
||||
await eval_repo.create_benchmark(payload)
|
||||
|
||||
async def _persist_file(self, file_id: str) -> None:
|
||||
"""只保存单个文件到数据库，避免全量遍历"""
|
||||
from yuxi.repositories.knowledge_file_repository import KnowledgeFileRepository
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import json
|
||||
import random
|
||||
from collections.abc import AsyncIterator, Callable
|
||||
@ -8,6 +9,9 @@ import json_repair
|
||||
from yuxi.models import select_model
|
||||
from yuxi.utils import logger
|
||||
|
||||
DEFAULT_BENCHMARK_GENERATION_CONCURRENCY = 10
|
||||
MAX_BENCHMARK_GENERATION_CONCURRENCY = 20
|
||||
|
||||
|
||||
async def collect_kb_chunks(kb_instance: Any, db_id: str) -> list[dict[str, Any]]:
|
||||
chunks = []
|
||||
@ -34,6 +38,12 @@ def clamp_neighbors_count(neighbors_count: int) -> int:
|
||||
return min(max(neighbors_count, 0), 10)
|
||||
|
||||
|
||||
def normalize_generation_concurrency_count(value: Any) -> int:
|
||||
if value in (None, ""):
|
||||
return DEFAULT_BENCHMARK_GENERATION_CONCURRENCY
|
||||
return min(max(1, int(value)), MAX_BENCHMARK_GENERATION_CONCURRENCY)
|
||||
|
||||
|
||||
def _is_anchor_chunk(candidate: dict[str, Any], anchor_chunk: dict[str, Any]) -> bool:
|
||||
metadata = candidate.get("metadata") or {}
|
||||
candidate_id = metadata.get("chunk_id")
|
||||
@ -99,6 +109,46 @@ def build_benchmark_generation_prompt(ctx_items: list[tuple[str, str]]) -> str:
|
||||
)
|
||||
|
||||
|
||||
async def _generate_benchmark_item_once(
|
||||
*,
|
||||
kb_instance: Any,
|
||||
db_id: str,
|
||||
all_chunks: list[dict[str, Any]],
|
||||
llm: Any,
|
||||
context_count: int,
|
||||
) -> dict[str, Any] | None:
|
||||
anchor_chunk = all_chunks[random.randrange(len(all_chunks))]
|
||||
neighbor_chunks = await select_neighbor_chunks_by_kb_query(
|
||||
kb_instance=kb_instance,
|
||||
db_id=db_id,
|
||||
anchor_chunk=anchor_chunk,
|
||||
neighbors_count=context_count - 1,
|
||||
)
|
||||
ctx_chunks = [anchor_chunk] + neighbor_chunks
|
||||
ctx_items = [(chunk["id"], chunk["content"]) for chunk in ctx_chunks]
|
||||
allowed_ids = {cid for cid, _ in ctx_items}
|
||||
|
||||
try:
|
||||
resp = await llm.call(build_benchmark_generation_prompt(ctx_items), False)
|
||||
obj = json_repair.loads(resp.content if resp else "")
|
||||
query = obj.get("query")
|
||||
answer = obj.get("gold_answer")
|
||||
gold_ids = obj.get("gold_chunk_ids")
|
||||
if not query or not answer or not isinstance(gold_ids, list):
|
||||
logger.warning(f"Generated JSON missing fields or invalid format: {obj}")
|
||||
return None
|
||||
|
||||
gold_ids = [str(item) for item in gold_ids if str(item) in allowed_ids]
|
||||
if not gold_ids:
|
||||
logger.warning("Generated gold_chunk_ids not found in allowed context")
|
||||
return None
|
||||
|
||||
return {"query": query, "gold_chunk_ids": gold_ids, "gold_answer": answer}
|
||||
except Exception as e:
|
||||
logger.warning(f"Benchmark generation failed for one item: {e}")
|
||||
return None
|
||||
|
||||
|
||||
async def iter_generated_benchmark_items(
|
||||
*,
|
||||
kb_instance: Any,
|
||||
@ -106,7 +156,9 @@ async def iter_generated_benchmark_items(
|
||||
count: int,
|
||||
neighbors_count: int,
|
||||
llm_model_spec: str | None,
|
||||
concurrency_count: int = DEFAULT_BENCHMARK_GENERATION_CONCURRENCY,
|
||||
progress_cb: Callable[[int, str], Any] | None = None,
|
||||
cancel_cb: Callable[[], Any] | None = None,
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
if progress_cb:
|
||||
await progress_cb(5, "加载chunks")
|
||||
@ -122,46 +174,71 @@ async def iter_generated_benchmark_items(
|
||||
raise ValueError("llm_model_spec 不能为空")
|
||||
llm = select_model(model_spec=llm_model_spec)
|
||||
context_count = max(clamp_neighbors_count(neighbors_count), 1)
|
||||
generated = 0
|
||||
attempts = 0
|
||||
max_attempts = max(count * 5, 50)
|
||||
worker_count = normalize_generation_concurrency_count(concurrency_count)
|
||||
actual_worker_count = min(worker_count, max(count, 1), max_attempts)
|
||||
generated = 0
|
||||
results: list[tuple[int, dict[str, Any]]] = []
|
||||
state_lock = asyncio.Lock()
|
||||
queue: asyncio.Queue[int] = asyncio.Queue()
|
||||
|
||||
while generated < count and attempts < max_attempts:
|
||||
attempts += 1
|
||||
anchor_chunk = all_chunks[random.randrange(len(all_chunks))]
|
||||
neighbor_chunks = await select_neighbor_chunks_by_kb_query(
|
||||
kb_instance=kb_instance,
|
||||
db_id=db_id,
|
||||
anchor_chunk=anchor_chunk,
|
||||
neighbors_count=context_count - 1,
|
||||
)
|
||||
ctx_chunks = [anchor_chunk] + neighbor_chunks
|
||||
ctx_items = [(chunk["id"], chunk["content"]) for chunk in ctx_chunks]
|
||||
allowed_ids = {cid for cid, _ in ctx_items}
|
||||
for attempt_no in range(max_attempts):
|
||||
queue.put_nowait(attempt_no)
|
||||
|
||||
try:
|
||||
resp = await llm.call(build_benchmark_generation_prompt(ctx_items), False)
|
||||
obj = json_repair.loads(resp.content if resp else "")
|
||||
query = obj.get("query")
|
||||
answer = obj.get("gold_answer")
|
||||
gold_ids = obj.get("gold_chunk_ids")
|
||||
if not query or not answer or not isinstance(gold_ids, list):
|
||||
logger.warning(f"Generated JSON missing fields or invalid format: {obj}")
|
||||
continue
|
||||
async def worker() -> None:
|
||||
nonlocal generated
|
||||
while True:
|
||||
if cancel_cb:
|
||||
await cancel_cb()
|
||||
async with state_lock:
|
||||
if generated >= count:
|
||||
return
|
||||
try:
|
||||
attempt_no = queue.get_nowait()
|
||||
except asyncio.QueueEmpty:
|
||||
return
|
||||
try:
|
||||
item = await _generate_benchmark_item_once(
|
||||
kb_instance=kb_instance,
|
||||
db_id=db_id,
|
||||
all_chunks=all_chunks,
|
||||
llm=llm,
|
||||
context_count=context_count,
|
||||
)
|
||||
if item is None:
|
||||
continue
|
||||
progress = None
|
||||
message = None
|
||||
async with state_lock:
|
||||
if generated >= count:
|
||||
continue
|
||||
generated += 1
|
||||
results.append((attempt_no, item))
|
||||
if progress_cb:
|
||||
progress = int(99 * generated / max(count, 1))
|
||||
message = f"已生成 {generated}/{count}"
|
||||
if progress_cb:
|
||||
await progress_cb(progress, message)
|
||||
finally:
|
||||
queue.task_done()
|
||||
|
||||
gold_ids = [str(item) for item in gold_ids if str(item) in allowed_ids]
|
||||
if not gold_ids:
|
||||
logger.warning("Generated gold_chunk_ids not found in allowed context")
|
||||
continue
|
||||
workers = [asyncio.create_task(worker()) for _ in range(actual_worker_count)]
|
||||
try:
|
||||
await asyncio.gather(*workers)
|
||||
except asyncio.CancelledError:
|
||||
for task in workers:
|
||||
task.cancel()
|
||||
await asyncio.gather(*workers, return_exceptions=True)
|
||||
raise
|
||||
except Exception:
|
||||
for task in workers:
|
||||
task.cancel()
|
||||
await asyncio.gather(*workers, return_exceptions=True)
|
||||
raise
|
||||
|
||||
generated += 1
|
||||
if progress_cb:
|
||||
await progress_cb(0 + int(99 * generated / max(count, 1)), f"已生成 {generated}/{count}")
|
||||
yield {"query": query, "gold_chunk_ids": gold_ids, "gold_answer": answer}
|
||||
except Exception as e:
|
||||
logger.warning(f"Benchmark generation failed for one item: {e}")
|
||||
continue
|
||||
for _, item in sorted(results, key=lambda pair: pair[0]):
|
||||
yield item
|
||||
|
||||
|
||||
def dump_benchmark_item(item: dict[str, Any]) -> str:
|
||||
return json.dumps(item, ensure_ascii=False) + "\n"
|
||||
return json.dumps(item, ensure_ascii=False, separators=(",", ":")) + "\n"
|
||||
|
||||
@ -2,73 +2,37 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy import delete, func, select
|
||||
|
||||
from yuxi.storage.postgres.manager import pg_manager
|
||||
from yuxi.storage.postgres.models_knowledge import EvaluationBenchmark, EvaluationResult, EvaluationResultDetail
|
||||
from yuxi.storage.postgres.models_knowledge import (
|
||||
EvaluationDataset,
|
||||
EvaluationDatasetItem,
|
||||
EvaluationRun,
|
||||
EvaluationRunItem,
|
||||
)
|
||||
|
||||
|
||||
class EvaluationRepository:
|
||||
async def get_all_benchmarks(self) -> list[EvaluationBenchmark]:
|
||||
"""获取所有评估基准"""
|
||||
async def create_dataset(self, dataset_data: dict[str, Any]) -> EvaluationDataset:
|
||||
dataset = EvaluationDataset(**dataset_data)
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(select(EvaluationBenchmark))
|
||||
return list(result.scalars().all())
|
||||
session.add(dataset)
|
||||
return dataset
|
||||
|
||||
async def create_benchmark(self, data: dict[str, Any]) -> EvaluationBenchmark:
|
||||
benchmark = EvaluationBenchmark(**data)
|
||||
async def create_dataset_with_items(
|
||||
self, dataset_data: dict[str, Any], items_data: list[dict[str, Any]]
|
||||
) -> EvaluationDataset:
|
||||
dataset = EvaluationDataset(**dataset_data)
|
||||
items = [EvaluationDatasetItem(**item) for item in items_data]
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
session.add(benchmark)
|
||||
return benchmark
|
||||
session.add(dataset)
|
||||
session.add_all(items)
|
||||
return dataset
|
||||
|
||||
async def get_benchmark(self, benchmark_id: str) -> EvaluationBenchmark | None:
|
||||
async def update_dataset(self, dataset_id: str, data: dict[str, Any]) -> EvaluationDataset | None:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(
|
||||
select(EvaluationBenchmark).where(EvaluationBenchmark.benchmark_id == benchmark_id)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def list_benchmarks(self, db_id: str) -> list[EvaluationBenchmark]:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(
|
||||
select(EvaluationBenchmark)
|
||||
.where(EvaluationBenchmark.db_id == db_id)
|
||||
.order_by(EvaluationBenchmark.created_at.desc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def delete_benchmark(self, benchmark_id: str) -> None:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(
|
||||
select(EvaluationBenchmark).where(EvaluationBenchmark.benchmark_id == benchmark_id)
|
||||
)
|
||||
record = result.scalar_one_or_none()
|
||||
if record is not None:
|
||||
await session.delete(record)
|
||||
|
||||
async def create_result(self, data: dict[str, Any]) -> EvaluationResult:
|
||||
result_row = EvaluationResult(**data)
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
session.add(result_row)
|
||||
return result_row
|
||||
|
||||
async def get_result(self, task_id: str) -> EvaluationResult | None:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(select(EvaluationResult).where(EvaluationResult.task_id == task_id))
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def list_results(self, db_id: str) -> list[EvaluationResult]:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(
|
||||
select(EvaluationResult)
|
||||
.where(EvaluationResult.db_id == db_id)
|
||||
.order_by(EvaluationResult.started_at.desc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def update_result(self, task_id: str, data: dict[str, Any]) -> EvaluationResult | None:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(select(EvaluationResult).where(EvaluationResult.task_id == task_id))
|
||||
result = await session.execute(select(EvaluationDataset).where(EvaluationDataset.dataset_id == dataset_id))
|
||||
record = result.scalar_one_or_none()
|
||||
if record is None:
|
||||
return None
|
||||
@ -76,43 +40,134 @@ class EvaluationRepository:
|
||||
setattr(record, key, value)
|
||||
return record
|
||||
|
||||
async def delete_result(self, task_id: str) -> None:
|
||||
async def add_dataset_items(self, items_data: list[dict[str, Any]]) -> None:
|
||||
items = [EvaluationDatasetItem(**item) for item in items_data]
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
await session.execute(delete(EvaluationResultDetail).where(EvaluationResultDetail.task_id == task_id))
|
||||
result = await session.execute(select(EvaluationResult).where(EvaluationResult.task_id == task_id))
|
||||
session.add_all(items)
|
||||
|
||||
async def get_dataset(self, dataset_id: str) -> EvaluationDataset | None:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(select(EvaluationDataset).where(EvaluationDataset.dataset_id == dataset_id))
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def list_datasets(self, db_id: str) -> list[EvaluationDataset]:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(
|
||||
select(EvaluationDataset)
|
||||
.where(EvaluationDataset.db_id == db_id)
|
||||
.order_by(EvaluationDataset.created_at.desc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def list_dataset_items(
|
||||
self, dataset_id: str, offset: int = 0, limit: int = 100
|
||||
) -> list[EvaluationDatasetItem]:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(
|
||||
select(EvaluationDatasetItem)
|
||||
.where(EvaluationDatasetItem.dataset_id == dataset_id)
|
||||
.order_by(EvaluationDatasetItem.item_index.asc())
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def count_dataset_items(self, dataset_id: str) -> int:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(
|
||||
select(func.count(EvaluationDatasetItem.id)).where(EvaluationDatasetItem.dataset_id == dataset_id)
|
||||
)
|
||||
return int(result.scalar() or 0)
|
||||
|
||||
async def list_all_dataset_items(self, dataset_id: str) -> list[EvaluationDatasetItem]:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(
|
||||
select(EvaluationDatasetItem)
|
||||
.where(EvaluationDatasetItem.dataset_id == dataset_id)
|
||||
.order_by(EvaluationDatasetItem.item_index.asc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def delete_dataset(self, dataset_id: str) -> None:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(select(EvaluationDataset).where(EvaluationDataset.dataset_id == dataset_id))
|
||||
record = result.scalar_one_or_none()
|
||||
if record is not None:
|
||||
await session.delete(record)
|
||||
|
||||
async def upsert_result_detail(
|
||||
self, task_id: str, query_index: int, data: dict[str, Any]
|
||||
) -> EvaluationResultDetail:
|
||||
async def create_run(self, data: dict[str, Any]) -> EvaluationRun:
|
||||
run = EvaluationRun(**data)
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
session.add(run)
|
||||
return run
|
||||
|
||||
async def get_run(self, run_id: str) -> EvaluationRun | None:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(select(EvaluationRun).where(EvaluationRun.run_id == run_id))
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def list_runs(self, db_id: str) -> list[EvaluationRun]:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(
|
||||
select(EvaluationResultDetail).where(
|
||||
(EvaluationResultDetail.task_id == task_id) & (EvaluationResultDetail.query_index == query_index)
|
||||
select(EvaluationRun).where(EvaluationRun.db_id == db_id).order_by(EvaluationRun.started_at.desc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def update_run(self, run_id: str, data: dict[str, Any]) -> EvaluationRun | None:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(select(EvaluationRun).where(EvaluationRun.run_id == run_id))
|
||||
record = result.scalar_one_or_none()
|
||||
if record is None:
|
||||
return None
|
||||
for key, value in data.items():
|
||||
setattr(record, key, value)
|
||||
return record
|
||||
|
||||
async def delete_run(self, run_id: str) -> None:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
await session.execute(delete(EvaluationRunItem).where(EvaluationRunItem.run_id == run_id))
|
||||
result = await session.execute(select(EvaluationRun).where(EvaluationRun.run_id == run_id))
|
||||
record = result.scalar_one_or_none()
|
||||
if record is not None:
|
||||
await session.delete(record)
|
||||
|
||||
async def upsert_run_item(self, run_id: str, item_index: int, data: dict[str, Any]) -> EvaluationRunItem:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(
|
||||
select(EvaluationRunItem).where(
|
||||
(EvaluationRunItem.run_id == run_id) & (EvaluationRunItem.item_index == item_index)
|
||||
)
|
||||
)
|
||||
record = result.scalar_one_or_none()
|
||||
if record is None:
|
||||
record = EvaluationResultDetail(task_id=task_id, query_index=query_index, **data)
|
||||
record = EvaluationRunItem(run_id=run_id, item_index=item_index, **data)
|
||||
session.add(record)
|
||||
return record
|
||||
for key, value in data.items():
|
||||
setattr(record, key, value)
|
||||
return record
|
||||
|
||||
async def list_result_details(self, task_id: str) -> list[EvaluationResultDetail]:
|
||||
async def list_run_items(self, run_id: str, offset: int = 0, limit: int = 100) -> list[EvaluationRunItem]:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(
|
||||
select(EvaluationResultDetail)
|
||||
.where(EvaluationResultDetail.task_id == task_id)
|
||||
.order_by(EvaluationResultDetail.query_index.asc())
|
||||
select(EvaluationRunItem)
|
||||
.where(EvaluationRunItem.run_id == run_id)
|
||||
.order_by(EvaluationRunItem.item_index.asc())
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def count_run_items(self, run_id: str) -> int:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
result = await session.execute(
|
||||
select(func.count(EvaluationRunItem.id)).where(EvaluationRunItem.run_id == run_id)
|
||||
)
|
||||
return int(result.scalar() or 0)
|
||||
|
||||
async def delete_all(self) -> None:
|
||||
async with pg_manager.get_async_session_context() as session:
|
||||
await session.execute(delete(EvaluationResultDetail))
|
||||
await session.execute(delete(EvaluationResult))
|
||||
await session.execute(delete(EvaluationBenchmark))
|
||||
await session.execute(delete(EvaluationRunItem))
|
||||
await session.execute(delete(EvaluationRun))
|
||||
await session.execute(delete(EvaluationDatasetItem))
|
||||
await session.execute(delete(EvaluationDataset))
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -148,21 +148,85 @@ class PostgresManager(metaclass=SingletonMeta):
|
||||
"ALTER TABLE IF EXISTS knowledge_files ADD COLUMN IF NOT EXISTS created_by VARCHAR(64)",
|
||||
"ALTER TABLE IF EXISTS knowledge_files ADD COLUMN IF NOT EXISTS updated_by VARCHAR(64)",
|
||||
"ALTER TABLE IF EXISTS knowledge_files ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ",
|
||||
"ALTER TABLE IF EXISTS evaluation_benchmarks ADD COLUMN IF NOT EXISTS data_file_path VARCHAR(1024)",
|
||||
"ALTER TABLE IF EXISTS evaluation_benchmarks ADD COLUMN IF NOT EXISTS created_by VARCHAR(64)",
|
||||
"ALTER TABLE IF EXISTS evaluation_benchmarks ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ",
|
||||
"ALTER TABLE IF EXISTS evaluation_results ADD COLUMN IF NOT EXISTS metrics JSONB",
|
||||
"ALTER TABLE IF EXISTS evaluation_results ADD COLUMN IF NOT EXISTS overall_score DOUBLE PRECISION",
|
||||
"ALTER TABLE IF EXISTS evaluation_results ADD COLUMN IF NOT EXISTS total_questions INTEGER",
|
||||
"ALTER TABLE IF EXISTS evaluation_results ADD COLUMN IF NOT EXISTS completed_questions INTEGER",
|
||||
"ALTER TABLE IF EXISTS evaluation_results ADD COLUMN IF NOT EXISTS started_at TIMESTAMPTZ",
|
||||
"ALTER TABLE IF EXISTS evaluation_results ADD COLUMN IF NOT EXISTS completed_at TIMESTAMPTZ",
|
||||
"ALTER TABLE IF EXISTS evaluation_results ADD COLUMN IF NOT EXISTS created_by VARCHAR(64)",
|
||||
"ALTER TABLE IF EXISTS evaluation_result_details ADD COLUMN IF NOT EXISTS gold_chunk_ids JSONB",
|
||||
"ALTER TABLE IF EXISTS evaluation_result_details ADD COLUMN IF NOT EXISTS gold_answer TEXT",
|
||||
"ALTER TABLE IF EXISTS evaluation_result_details ADD COLUMN IF NOT EXISTS generated_answer TEXT",
|
||||
"ALTER TABLE IF EXISTS evaluation_result_details ADD COLUMN IF NOT EXISTS retrieved_chunks JSONB",
|
||||
"ALTER TABLE IF EXISTS evaluation_result_details ADD COLUMN IF NOT EXISTS metrics JSONB",
|
||||
"ALTER TABLE IF EXISTS evaluation_datasets ADD COLUMN IF NOT EXISTS created_by VARCHAR(64)",
|
||||
"ALTER TABLE IF EXISTS evaluation_datasets ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ",
|
||||
"ALTER TABLE IF EXISTS evaluation_datasets ADD COLUMN IF NOT EXISTS build_metadata JSONB",
|
||||
"ALTER TABLE IF EXISTS evaluation_runs ADD COLUMN IF NOT EXISTS metrics JSONB",
|
||||
"ALTER TABLE IF EXISTS evaluation_runs ADD COLUMN IF NOT EXISTS overall_score DOUBLE PRECISION",
|
||||
"ALTER TABLE IF EXISTS evaluation_runs ADD COLUMN IF NOT EXISTS total_items INTEGER",
|
||||
"ALTER TABLE IF EXISTS evaluation_runs ADD COLUMN IF NOT EXISTS completed_items INTEGER",
|
||||
"ALTER TABLE IF EXISTS evaluation_runs ADD COLUMN IF NOT EXISTS started_at TIMESTAMPTZ",
|
||||
"ALTER TABLE IF EXISTS evaluation_runs ADD COLUMN IF NOT EXISTS completed_at TIMESTAMPTZ",
|
||||
"ALTER TABLE IF EXISTS evaluation_runs ADD COLUMN IF NOT EXISTS created_by VARCHAR(64)",
|
||||
"ALTER TABLE IF EXISTS evaluation_run_items ADD COLUMN IF NOT EXISTS gold_chunk_ids JSONB",
|
||||
"ALTER TABLE IF EXISTS evaluation_run_items ADD COLUMN IF NOT EXISTS gold_answer TEXT",
|
||||
"ALTER TABLE IF EXISTS evaluation_run_items ADD COLUMN IF NOT EXISTS generated_answer TEXT",
|
||||
"ALTER TABLE IF EXISTS evaluation_run_items ADD COLUMN IF NOT EXISTS retrieved_chunks JSONB",
|
||||
"ALTER TABLE IF EXISTS evaluation_run_items ADD COLUMN IF NOT EXISTS metrics JSONB",
|
||||
"ALTER TABLE IF EXISTS evaluation_run_items ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ",
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS evaluation_datasets (
|
||||
id SERIAL PRIMARY KEY,
|
||||
dataset_id VARCHAR(64) NOT NULL UNIQUE,
|
||||
db_id VARCHAR(80) NOT NULL REFERENCES knowledge_bases(db_id) ON DELETE CASCADE,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
description TEXT,
|
||||
item_count INTEGER DEFAULT 0,
|
||||
has_gold_chunks BOOLEAN DEFAULT FALSE,
|
||||
has_gold_answers BOOLEAN DEFAULT FALSE,
|
||||
build_metadata JSONB,
|
||||
created_by VARCHAR(64),
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS evaluation_dataset_items (
|
||||
id SERIAL PRIMARY KEY,
|
||||
item_id VARCHAR(64) NOT NULL UNIQUE,
|
||||
dataset_id VARCHAR(64) NOT NULL REFERENCES evaluation_datasets(dataset_id) ON DELETE CASCADE,
|
||||
db_id VARCHAR(80) NOT NULL REFERENCES knowledge_bases(db_id) ON DELETE CASCADE,
|
||||
item_index INTEGER NOT NULL,
|
||||
query_text TEXT NOT NULL,
|
||||
gold_chunk_ids JSONB,
|
||||
gold_answer TEXT,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
CONSTRAINT uq_evaluation_dataset_items_dataset_index UNIQUE (dataset_id, item_index)
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS evaluation_runs (
|
||||
id SERIAL PRIMARY KEY,
|
||||
run_id VARCHAR(64) NOT NULL UNIQUE,
|
||||
db_id VARCHAR(80) NOT NULL REFERENCES knowledge_bases(db_id) ON DELETE CASCADE,
|
||||
dataset_id VARCHAR(64) REFERENCES evaluation_datasets(dataset_id) ON DELETE SET NULL,
|
||||
status VARCHAR(32) DEFAULT 'running',
|
||||
retrieval_config JSONB,
|
||||
metrics JSONB,
|
||||
overall_score DOUBLE PRECISION,
|
||||
total_items INTEGER DEFAULT 0,
|
||||
completed_items INTEGER DEFAULT 0,
|
||||
started_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
completed_at TIMESTAMPTZ,
|
||||
created_by VARCHAR(64)
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS evaluation_run_items (
|
||||
id SERIAL PRIMARY KEY,
|
||||
run_id VARCHAR(64) NOT NULL REFERENCES evaluation_runs(run_id) ON DELETE CASCADE,
|
||||
dataset_item_id VARCHAR(64) REFERENCES evaluation_dataset_items(item_id) ON DELETE SET NULL,
|
||||
item_index INTEGER NOT NULL,
|
||||
query_text TEXT NOT NULL,
|
||||
gold_chunk_ids JSONB,
|
||||
gold_answer TEXT,
|
||||
generated_answer TEXT,
|
||||
retrieved_chunks JSONB,
|
||||
metrics JSONB,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
CONSTRAINT uq_evaluation_run_items_run_index UNIQUE (run_id, item_index)
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS knowledge_chunks (
|
||||
id SERIAL PRIMARY KEY,
|
||||
@ -238,19 +302,25 @@ class PostgresManager(metaclass=SingletonMeta):
|
||||
# 扩展 db_id 字段长度以支持最长 75 字符的 ID（kb_private_ + 64字符hash）
|
||||
"ALTER TABLE IF EXISTS knowledge_bases ALTER COLUMN db_id TYPE VARCHAR(80)",
|
||||
"ALTER TABLE IF EXISTS knowledge_files ALTER COLUMN db_id TYPE VARCHAR(80)",
|
||||
"ALTER TABLE IF EXISTS evaluation_benchmarks ALTER COLUMN db_id TYPE VARCHAR(80)",
|
||||
"ALTER TABLE IF EXISTS evaluation_results ALTER COLUMN db_id TYPE VARCHAR(80)",
|
||||
"ALTER TABLE IF EXISTS evaluation_datasets ALTER COLUMN db_id TYPE VARCHAR(80)",
|
||||
"ALTER TABLE IF EXISTS evaluation_dataset_items ALTER COLUMN db_id TYPE VARCHAR(80)",
|
||||
"ALTER TABLE IF EXISTS evaluation_runs ALTER COLUMN db_id TYPE VARCHAR(80)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_kb_type ON knowledge_bases(kb_type)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_kb_name ON knowledge_bases(name)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_kf_db_id ON knowledge_files(db_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_kf_parent ON knowledge_files(parent_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_kf_status ON knowledge_files(status)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_kf_hash ON knowledge_files(content_hash)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_eb_db_id ON evaluation_benchmarks(db_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_er_db_id ON evaluation_results(db_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_er_status ON evaluation_results(status)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_er_started ON evaluation_results(started_at DESC)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_erd_task ON evaluation_result_details(task_id)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_evaluation_datasets_db_id ON evaluation_datasets(db_id)",
|
||||
(
|
||||
"CREATE INDEX IF NOT EXISTS ix_evaluation_dataset_items_dataset_index "
|
||||
"ON evaluation_dataset_items(dataset_id, item_index)"
|
||||
),
|
||||
"CREATE INDEX IF NOT EXISTS ix_evaluation_dataset_items_db_id ON evaluation_dataset_items(db_id)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_evaluation_runs_db_id ON evaluation_runs(db_id)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_evaluation_runs_status ON evaluation_runs(status)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_evaluation_runs_started ON evaluation_runs(started_at DESC)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_evaluation_run_items_run_index ON evaluation_run_items(run_id, item_index)",
|
||||
"CREATE UNIQUE INDEX IF NOT EXISTS uq_knowledge_chunks_chunk_id ON knowledge_chunks(chunk_id)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_knowledge_chunks_file_id ON knowledge_chunks(file_id)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_knowledge_chunks_db_id ON knowledge_chunks(db_id)",
|
||||
|
||||
@ -187,68 +187,101 @@ class KnowledgeGraphTripleMention(Base):
|
||||
created_at = Column(DateTime(timezone=True), default=utc_now_naive)
|
||||
|
||||
|
||||
class EvaluationBenchmark(Base):
|
||||
"""评估基准模型"""
|
||||
class EvaluationDataset(Base):
|
||||
"""评估数据集模型"""
|
||||
|
||||
__tablename__ = "evaluation_benchmarks"
|
||||
__table_args__ = (UniqueConstraint("benchmark_id", name="uq_evaluation_benchmarks_benchmark_id"),)
|
||||
__tablename__ = "evaluation_datasets"
|
||||
__table_args__ = (UniqueConstraint("dataset_id", name="uq_evaluation_datasets_dataset_id"),)
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
benchmark_id = Column(String(64), unique=True, nullable=False, index=True)
|
||||
dataset_id = Column(String(64), unique=True, nullable=False, index=True)
|
||||
db_id = Column(String(80), ForeignKey("knowledge_bases.db_id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
name = Column(String(255), nullable=False)
|
||||
description = Column(Text)
|
||||
question_count = Column(Integer, default=0)
|
||||
item_count = Column(Integer, default=0)
|
||||
has_gold_chunks = Column(Boolean, default=False)
|
||||
has_gold_answers = Column(Boolean, default=False)
|
||||
data_file_path = Column(String(1024))
|
||||
build_metadata = Column(JSON_VALUE)
|
||||
created_by = Column(String(64))
|
||||
created_at = Column(DateTime(timezone=True), default=utc_now_naive)
|
||||
updated_at = Column(DateTime(timezone=True), default=utc_now_naive, onupdate=utc_now_naive)
|
||||
|
||||
|
||||
class EvaluationResult(Base):
|
||||
"""评估结果模型"""
|
||||
class EvaluationDatasetItem(Base):
|
||||
"""评估数据集题目模型"""
|
||||
|
||||
__tablename__ = "evaluation_results"
|
||||
__table_args__ = (UniqueConstraint("task_id", name="uq_evaluation_results_task_id"),)
|
||||
__tablename__ = "evaluation_dataset_items"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("item_id", name="uq_evaluation_dataset_items_item_id"),
|
||||
UniqueConstraint("dataset_id", "item_index", name="uq_evaluation_dataset_items_dataset_index"),
|
||||
Index("ix_evaluation_dataset_items_dataset_index", "dataset_id", "item_index"),
|
||||
)
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
task_id = Column(String(64), unique=True, nullable=False, index=True)
|
||||
db_id = Column(String(80), ForeignKey("knowledge_bases.db_id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
benchmark_id = Column(
|
||||
item_id = Column(String(64), unique=True, nullable=False, index=True)
|
||||
dataset_id = Column(
|
||||
String(64),
|
||||
ForeignKey("evaluation_benchmarks.benchmark_id", ondelete="SET NULL"),
|
||||
ForeignKey("evaluation_datasets.dataset_id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
db_id = Column(String(80), ForeignKey("knowledge_bases.db_id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
item_index = Column(Integer, nullable=False)
|
||||
query_text = Column(Text, nullable=False)
|
||||
gold_chunk_ids = Column(JSON_VALUE)
|
||||
gold_answer = Column(Text)
|
||||
created_at = Column(DateTime(timezone=True), default=utc_now_naive)
|
||||
|
||||
|
||||
class EvaluationRun(Base):
|
||||
"""评估运行模型"""
|
||||
|
||||
__tablename__ = "evaluation_runs"
|
||||
__table_args__ = (UniqueConstraint("run_id", name="uq_evaluation_runs_run_id"),)
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
run_id = Column(String(64), unique=True, nullable=False, index=True)
|
||||
db_id = Column(String(80), ForeignKey("knowledge_bases.db_id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
dataset_id = Column(
|
||||
String(64),
|
||||
ForeignKey("evaluation_datasets.dataset_id", ondelete="SET NULL"),
|
||||
index=True,
|
||||
)
|
||||
status = Column(String(32), default="running", index=True)
|
||||
retrieval_config = Column(JSON_VALUE)
|
||||
metrics = Column(JSON_VALUE)
|
||||
overall_score = Column(Float)
|
||||
total_questions = Column(Integer, default=0)
|
||||
completed_questions = Column(Integer, default=0)
|
||||
total_items = Column(Integer, default=0)
|
||||
completed_items = Column(Integer, default=0)
|
||||
started_at = Column(DateTime(timezone=True), default=utc_now_naive, index=True)
|
||||
completed_at = Column(DateTime(timezone=True))
|
||||
created_by = Column(String(64))
|
||||
|
||||
|
||||
class EvaluationResultDetail(Base):
|
||||
"""评估结果详情模型"""
|
||||
class EvaluationRunItem(Base):
|
||||
"""评估逐题结果模型"""
|
||||
|
||||
__tablename__ = "evaluation_result_details"
|
||||
__table_args__ = (UniqueConstraint("task_id", "query_index", name="uq_evaluation_result_details_task_query"),)
|
||||
__tablename__ = "evaluation_run_items"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("run_id", "item_index", name="uq_evaluation_run_items_run_index"),
|
||||
Index("ix_evaluation_run_items_run_index", "run_id", "item_index"),
|
||||
)
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
task_id = Column(
|
||||
run_id = Column(
|
||||
String(64),
|
||||
ForeignKey("evaluation_results.task_id", ondelete="CASCADE"),
|
||||
ForeignKey("evaluation_runs.run_id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
query_index = Column(Integer, nullable=False)
|
||||
dataset_item_id = Column(
|
||||
String(64), ForeignKey("evaluation_dataset_items.item_id", ondelete="SET NULL"), index=True
|
||||
)
|
||||
item_index = Column(Integer, nullable=False)
|
||||
query_text = Column(Text, nullable=False)
|
||||
gold_chunk_ids = Column(JSON_VALUE)
|
||||
gold_answer = Column(Text)
|
||||
generated_answer = Column(Text)
|
||||
retrieved_chunks = Column(JSON_VALUE)
|
||||
metrics = Column(JSON_VALUE)
|
||||
created_at = Column(DateTime(timezone=True), default=utc_now_naive)
|
||||
|
||||
@ -1,226 +1,261 @@
|
||||
import traceback
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends, File, Form, Body, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
from yuxi.storage.postgres.models_business import User
|
||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile
|
||||
from fastapi.responses import Response
|
||||
from pydantic import BaseModel, Field
|
||||
from server.utils.auth_middleware import get_admin_user
|
||||
from yuxi.knowledge.eval.benchmark_generation import (
|
||||
DEFAULT_BENCHMARK_GENERATION_CONCURRENCY,
|
||||
MAX_BENCHMARK_GENERATION_CONCURRENCY,
|
||||
)
|
||||
from yuxi.storage.postgres.models_business import User
|
||||
from yuxi.utils import logger
|
||||
|
||||
# 创建路由器
|
||||
|
||||
evaluation = APIRouter(prefix="/evaluation", tags=["evaluation"])
|
||||
|
||||
|
||||
# 移除旧详情接口，统一使用带 db_id 的接口
|
||||
# ============================================================================
|
||||
# 评估基准
|
||||
# ============================================================================
|
||||
class GenerateDatasetRequest(BaseModel):
|
||||
name: str = Field(default="自动生成评估数据集", min_length=1, max_length=100)
|
||||
description: str = ""
|
||||
count: int = Field(default=10, ge=1, le=100)
|
||||
neighbors_count: int = Field(default=1, ge=0, le=10)
|
||||
concurrency_count: int = Field(
|
||||
default=DEFAULT_BENCHMARK_GENERATION_CONCURRENCY,
|
||||
ge=1,
|
||||
le=MAX_BENCHMARK_GENERATION_CONCURRENCY,
|
||||
)
|
||||
llm_model_spec: str = Field(..., min_length=1)
|
||||
|
||||
|
||||
@evaluation.get("/databases/{db_id}/benchmarks/{benchmark_id}")
|
||||
async def get_evaluation_benchmark_by_db(
|
||||
db_id: str, benchmark_id: str, page: int = 1, page_size: int = 10, current_user: User = Depends(get_admin_user)
|
||||
):
|
||||
"""根据 db_id 获取评估基准详情（支持分页）"""
|
||||
from yuxi.services.evaluation_service import EvaluationService
|
||||
|
||||
try:
|
||||
# 验证分页参数
|
||||
if page < 1:
|
||||
raise HTTPException(status_code=400, detail="页码必须大于0")
|
||||
if page_size < 1 or page_size > 100:
|
||||
raise HTTPException(status_code=400, detail="每页大小必须在1-100之间")
|
||||
|
||||
service = EvaluationService()
|
||||
benchmark = await service.get_benchmark_detail_by_db(db_id, benchmark_id, page, page_size)
|
||||
return {"message": "success", "data": benchmark}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"获取评估基准详情失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"获取评估基准详情失败: {str(e)}")
|
||||
class RunEvaluationRequest(BaseModel):
|
||||
dataset_id: str = Field(..., min_length=1)
|
||||
retrieval_config: dict[str, Any] = Field(default_factory=dict, alias="model_config")
|
||||
|
||||
|
||||
@evaluation.delete("/benchmarks/{benchmark_id}")
|
||||
async def delete_evaluation_benchmark(benchmark_id: str, current_user: User = Depends(get_admin_user)):
|
||||
"""删除评估基准"""
|
||||
from yuxi.services.evaluation_service import EvaluationService
|
||||
|
||||
try:
|
||||
service = EvaluationService()
|
||||
await service.delete_benchmark(benchmark_id)
|
||||
return {"message": "success", "data": None}
|
||||
except Exception as e:
|
||||
logger.error(f"删除评估基准失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"删除评估基准失败: {str(e)}")
|
||||
|
||||
|
||||
@evaluation.get("/benchmarks/{benchmark_id}/download")
|
||||
async def download_evaluation_benchmark(benchmark_id: str, current_user: User = Depends(get_admin_user)):
|
||||
"""下载评估基准文件"""
|
||||
from yuxi.services.evaluation_service import EvaluationService
|
||||
|
||||
try:
|
||||
service = EvaluationService()
|
||||
download_info = await service.get_benchmark_download_info(benchmark_id)
|
||||
return FileResponse(
|
||||
path=download_info["file_path"],
|
||||
filename=download_info["filename"],
|
||||
media_type="application/x-ndjson",
|
||||
)
|
||||
except ValueError as e:
|
||||
if "not found" in str(e).lower():
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
logger.error(f"下载评估基准失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"下载评估基准失败: {str(e)}")
|
||||
except Exception as e:
|
||||
logger.error(f"下载评估基准失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"下载评估基准失败: {str(e)}")
|
||||
|
||||
|
||||
@evaluation.get("/databases/{db_id}/results/{task_id}")
|
||||
async def get_evaluation_results_by_db(
|
||||
db_id: str,
|
||||
task_id: str,
|
||||
page: int = 1,
|
||||
page_size: int = 20,
|
||||
error_only: bool = False,
|
||||
current_user: User = Depends(get_admin_user),
|
||||
):
|
||||
"""获取评估结果（带 db_id，支持分页）"""
|
||||
from yuxi.services.evaluation_service import EvaluationService
|
||||
|
||||
try:
|
||||
# 验证分页参数
|
||||
if page < 1:
|
||||
raise HTTPException(status_code=400, detail="页码必须大于0")
|
||||
if page_size < 1 or page_size > 100:
|
||||
raise HTTPException(status_code=400, detail="每页大小必须在1-100之间")
|
||||
|
||||
service = EvaluationService()
|
||||
results = await service.get_evaluation_results_by_db(
|
||||
db_id, task_id, page=page, page_size=page_size, error_only=error_only
|
||||
)
|
||||
return {"message": "success", "data": results}
|
||||
except Exception as e:
|
||||
logger.error(f"获取评估结果失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"获取评估结果失败: {str(e)}")
|
||||
|
||||
|
||||
@evaluation.delete("/databases/{db_id}/results/{task_id}")
|
||||
async def delete_evaluation_result_by_db(db_id: str, task_id: str, current_user: User = Depends(get_admin_user)):
|
||||
"""删除评估结果（带 db_id）"""
|
||||
from yuxi.services.evaluation_service import EvaluationService
|
||||
|
||||
try:
|
||||
service = EvaluationService()
|
||||
await service.delete_evaluation_result_by_db(db_id, task_id)
|
||||
return {"message": "success", "data": None}
|
||||
except Exception as e:
|
||||
logger.error(f"删除评估结果失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"删除评估结果失败: {str(e)}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# RAG评估
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@evaluation.post("/databases/{db_id}/benchmarks/upload")
|
||||
async def upload_evaluation_benchmark(
|
||||
@evaluation.post("/databases/{db_id}/datasets/upload")
|
||||
async def upload_evaluation_dataset(
|
||||
db_id: str,
|
||||
file: UploadFile = File(...),
|
||||
name: str = Form(...),
|
||||
description: str = Form(""),
|
||||
current_user: User = Depends(get_admin_user),
|
||||
):
|
||||
"""上传评估基准文件"""
|
||||
"""上传评估数据集"""
|
||||
from yuxi.services.evaluation_service import EvaluationService
|
||||
|
||||
try:
|
||||
# 验证文件格式
|
||||
if not file.filename.endswith(".jsonl"):
|
||||
raise HTTPException(status_code=400, detail="仅支持JSONL格式文件")
|
||||
|
||||
# 读取文件内容
|
||||
content = await file.read()
|
||||
|
||||
# 调用评估服务处理上传
|
||||
service = EvaluationService()
|
||||
result = await service.upload_benchmark(
|
||||
result = await service.upload_dataset(
|
||||
db_id=db_id,
|
||||
file_content=content,
|
||||
file_content=await file.read(),
|
||||
filename=file.filename,
|
||||
name=name,
|
||||
description=description,
|
||||
created_by=current_user.uid,
|
||||
)
|
||||
|
||||
return {"message": "success", "data": result}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"上传评估基准失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"上传评估基准失败: {str(e)}")
|
||||
logger.error(f"上传评估数据集失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"上传评估数据集失败: {str(e)}")
|
||||
|
||||
|
||||
@evaluation.get("/databases/{db_id}/benchmarks")
|
||||
async def get_evaluation_benchmarks(db_id: str, current_user: User = Depends(get_admin_user)):
|
||||
"""获取知识库的评估基准列表"""
|
||||
@evaluation.get("/databases/{db_id}/datasets")
|
||||
async def list_evaluation_datasets(db_id: str, current_user: User = Depends(get_admin_user)):
|
||||
"""获取知识库的评估数据集列表"""
|
||||
from yuxi.services.evaluation_service import EvaluationService
|
||||
|
||||
try:
|
||||
service = EvaluationService()
|
||||
benchmarks = await service.get_benchmarks(db_id)
|
||||
return {"message": "success", "data": benchmarks}
|
||||
datasets = await service.list_datasets(db_id)
|
||||
return {"message": "success", "data": datasets}
|
||||
except Exception as e:
|
||||
logger.error(f"获取评估基准列表失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"获取评估基准列表失败: {str(e)}")
|
||||
logger.error(f"获取评估数据集列表失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"获取评估数据集列表失败: {str(e)}")
|
||||
|
||||
|
||||
@evaluation.post("/databases/{db_id}/benchmarks/generate")
|
||||
async def generate_evaluation_benchmark(
|
||||
db_id: str, params: dict = Body(...), current_user: User = Depends(get_admin_user)
|
||||
@evaluation.get("/databases/{db_id}/datasets/{dataset_id}")
|
||||
async def get_evaluation_dataset(
|
||||
db_id: str, dataset_id: str, page: int = 1, page_size: int = 10, current_user: User = Depends(get_admin_user)
|
||||
):
|
||||
"""自动生成评估基准"""
|
||||
"""获取评估数据集详情"""
|
||||
from yuxi.services.evaluation_service import EvaluationService
|
||||
|
||||
try:
|
||||
if page < 1:
|
||||
raise HTTPException(status_code=400, detail="页码必须大于0")
|
||||
if page_size < 1 or page_size > 100:
|
||||
raise HTTPException(status_code=400, detail="每页大小必须在1-100之间")
|
||||
|
||||
service = EvaluationService()
|
||||
dataset = await service.get_dataset_detail(db_id, dataset_id, page, page_size)
|
||||
return {"message": "success", "data": dataset}
|
||||
except HTTPException:
|
||||
raise
|
||||
except ValueError as e:
|
||||
if "not found" in str(e).lower():
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"获取评估数据集详情失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"获取评估数据集详情失败: {str(e)}")
|
||||
|
||||
|
||||
@evaluation.get("/datasets/{dataset_id}/download")
|
||||
async def download_evaluation_dataset(dataset_id: str, current_user: User = Depends(get_admin_user)):
|
||||
"""导出评估数据集 JSONL"""
|
||||
from yuxi.services.evaluation_service import EvaluationService
|
||||
|
||||
try:
|
||||
service = EvaluationService()
|
||||
result = await service.generate_benchmark(db_id=db_id, params=params, created_by=current_user.uid)
|
||||
export_info = await service.export_dataset_jsonl(dataset_id)
|
||||
filename = export_info["filename"]
|
||||
return Response(
|
||||
content=export_info["content"].encode("utf-8"),
|
||||
media_type="application/x-ndjson",
|
||||
headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename)}"},
|
||||
)
|
||||
except ValueError as e:
|
||||
if "not found" in str(e).lower():
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"导出评估数据集失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"导出评估数据集失败: {str(e)}")
|
||||
|
||||
|
||||
@evaluation.delete("/datasets/{dataset_id}")
|
||||
async def delete_evaluation_dataset(dataset_id: str, current_user: User = Depends(get_admin_user)):
|
||||
"""删除评估数据集"""
|
||||
from yuxi.services.evaluation_service import EvaluationService
|
||||
|
||||
try:
|
||||
service = EvaluationService()
|
||||
await service.delete_dataset(dataset_id)
|
||||
return {"message": "success", "data": None}
|
||||
except ValueError as e:
|
||||
if "not found" in str(e).lower():
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"删除评估数据集失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"删除评估数据集失败: {str(e)}")
|
||||
|
||||
|
||||
@evaluation.post("/databases/{db_id}/datasets/generate")
|
||||
async def generate_evaluation_dataset(
|
||||
db_id: str, request: GenerateDatasetRequest, current_user: User = Depends(get_admin_user)
|
||||
):
|
||||
"""自动生成评估数据集"""
|
||||
from yuxi.services.evaluation_service import EvaluationService
|
||||
|
||||
try:
|
||||
service = EvaluationService()
|
||||
result = await service.generate_dataset(
|
||||
db_id=db_id,
|
||||
name=request.name,
|
||||
description=request.description,
|
||||
count=request.count,
|
||||
neighbors_count=request.neighbors_count,
|
||||
concurrency_count=request.concurrency_count,
|
||||
llm_model_spec=request.llm_model_spec,
|
||||
created_by=current_user.uid,
|
||||
)
|
||||
return {"message": "success", "data": result}
|
||||
except Exception as e:
|
||||
logger.error(f"生成评估基准失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"生成评估基准失败: {str(e)}")
|
||||
logger.error(f"生成评估数据集失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"生成评估数据集失败: {str(e)}")
|
||||
|
||||
|
||||
@evaluation.post("/databases/{db_id}/run")
|
||||
async def run_evaluation(db_id: str, params: dict = Body(...), current_user: User = Depends(get_admin_user)):
|
||||
@evaluation.post("/databases/{db_id}/runs")
|
||||
async def run_evaluation(db_id: str, request: RunEvaluationRequest, current_user: User = Depends(get_admin_user)):
|
||||
"""运行RAG评估"""
|
||||
from yuxi.services.evaluation_service import EvaluationService
|
||||
|
||||
try:
|
||||
service = EvaluationService()
|
||||
task_id = await service.run_evaluation(
|
||||
run_id = await service.run_evaluation(
|
||||
db_id=db_id,
|
||||
benchmark_id=params.get("benchmark_id"),
|
||||
model_config=params.get("model_config", {}),
|
||||
dataset_id=request.dataset_id,
|
||||
model_config=request.retrieval_config,
|
||||
created_by=current_user.uid,
|
||||
)
|
||||
return {"message": "success", "data": {"task_id": task_id}}
|
||||
return {"message": "success", "data": {"run_id": run_id}}
|
||||
except ValueError as e:
|
||||
if "not found" in str(e).lower():
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"启动评估失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"启动评估失败: {str(e)}")
|
||||
|
||||
|
||||
@evaluation.get("/databases/{db_id}/history")
|
||||
async def get_evaluation_history(db_id: str, current_user: User = Depends(get_admin_user)):
|
||||
"""获取知识库的评估历史记录"""
|
||||
@evaluation.get("/databases/{db_id}/runs")
|
||||
async def list_evaluation_runs(db_id: str, current_user: User = Depends(get_admin_user)):
|
||||
"""获取知识库评估运行历史"""
|
||||
from yuxi.services.evaluation_service import EvaluationService
|
||||
|
||||
try:
|
||||
service = EvaluationService()
|
||||
history = await service.get_evaluation_history(db_id)
|
||||
return {"message": "success", "data": history}
|
||||
runs = await service.list_runs(db_id)
|
||||
return {"message": "success", "data": runs}
|
||||
except Exception as e:
|
||||
logger.error(f"获取评估历史失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"获取评估历史失败: {str(e)}")
|
||||
logger.error(f"获取评估运行历史失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"获取评估运行历史失败: {str(e)}")
|
||||
|
||||
|
||||
@evaluation.get("/databases/{db_id}/runs/{run_id}")
|
||||
async def get_evaluation_run_results(
|
||||
db_id: str,
|
||||
run_id: str,
|
||||
page: int = 1,
|
||||
page_size: int = 20,
|
||||
error_only: bool = False,
|
||||
current_user: User = Depends(get_admin_user),
|
||||
):
|
||||
"""获取评估运行结果"""
|
||||
from yuxi.services.evaluation_service import EvaluationService
|
||||
|
||||
try:
|
||||
if page < 1:
|
||||
raise HTTPException(status_code=400, detail="页码必须大于0")
|
||||
if page_size < 1 or page_size > 100:
|
||||
raise HTTPException(status_code=400, detail="每页大小必须在1-100之间")
|
||||
|
||||
service = EvaluationService()
|
||||
results = await service.get_run_results(db_id, run_id, page=page, page_size=page_size, error_only=error_only)
|
||||
return {"message": "success", "data": results}
|
||||
except HTTPException:
|
||||
raise
|
||||
except ValueError as e:
|
||||
if "not found" in str(e).lower():
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"获取评估运行结果失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"获取评估运行结果失败: {str(e)}")
|
||||
|
||||
|
||||
@evaluation.delete("/databases/{db_id}/runs/{run_id}")
|
||||
async def delete_evaluation_run(db_id: str, run_id: str, current_user: User = Depends(get_admin_user)):
|
||||
"""删除评估运行"""
|
||||
from yuxi.services.evaluation_service import EvaluationService
|
||||
|
||||
try:
|
||||
service = EvaluationService()
|
||||
await service.delete_run(db_id, run_id)
|
||||
return {"message": "success", "data": None}
|
||||
except ValueError as e:
|
||||
if "not found" in str(e).lower():
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"删除评估运行失败: {e}, {traceback.format_exc()}")
|
||||
raise HTTPException(status_code=500, detail=f"删除评估运行失败: {str(e)}")
|
||||
|
||||
@ -22,7 +22,7 @@ async def lifespan(app: FastAPI):
|
||||
# 初始化数据库连接
|
||||
try:
|
||||
pg_manager.initialize()
|
||||
await pg_manager.create_business_tables()
|
||||
await pg_manager.create_tables()
|
||||
await pg_manager.ensure_business_schema()
|
||||
await pg_manager.ensure_knowledge_schema()
|
||||
except Exception as e:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user