肖泽涛
parent
ad2da86a85
commit
9c30c74d41
@ -13,8 +13,6 @@ services:
|
||||
- ./test:/app/test
|
||||
- ./scripts:/app/scripts
|
||||
- ./.env:/app/.env
|
||||
- ./pyproject.toml:/app/pyproject.toml
|
||||
- ./uv.lock:/app/uv.lock
|
||||
- ${MODEL_DIR:-./models}:/models # 使用默认值处理未定义的环境变量
|
||||
ports:
|
||||
- "5050:5050"
|
||||
|
||||
@ -68,7 +68,6 @@ dependencies = [
|
||||
"torchvision==0.23",
|
||||
"docling>=2.68.0",
|
||||
"loguru>=0.7.3",
|
||||
"langfuse>=2.0.0",
|
||||
"redis>=5.2.0",
|
||||
]
|
||||
[tool.ruff]
|
||||
@ -120,7 +119,3 @@ test = [
|
||||
"pytest-httpx>=0.32.0",
|
||||
"pytest-cov>=6.0.0",
|
||||
]
|
||||
eval = [
|
||||
"datasets>=3.0.0",
|
||||
"huggingface-hub>=0.25.0",
|
||||
]
|
||||
|
||||
@ -14,7 +14,6 @@ from src import config as sys_config
|
||||
from src.agents.common.context import BaseContext
|
||||
from src.services.skill_resolver import get_skill_options_from_db
|
||||
from src.utils import logger
|
||||
from src.utils.langfuse_integration import get_langfuse_callback
|
||||
|
||||
|
||||
class BaseAgent:
|
||||
@ -76,13 +75,7 @@ class BaseAgent:
|
||||
if isinstance(agent_config, dict):
|
||||
context.update(agent_config)
|
||||
context.update(input_context or {})
|
||||
|
||||
stream_config = {}
|
||||
langfuse_handler = get_langfuse_callback()
|
||||
if langfuse_handler:
|
||||
stream_config["callbacks"] = [langfuse_handler]
|
||||
|
||||
for event in graph.astream({"messages": messages}, stream_mode="values", context=context, config=stream_config):
|
||||
for event in graph.astream({"messages": messages}, stream_mode="values", context=context):
|
||||
yield event["messages"]
|
||||
|
||||
async def stream_messages(self, messages: list[str], input_context=None, **kwargs):
|
||||
@ -99,9 +92,6 @@ class BaseAgent:
|
||||
"configurable": {"thread_id": context.thread_id, "user_id": context.user_id},
|
||||
"recursion_limit": 300,
|
||||
}
|
||||
langfuse_handler = get_langfuse_callback()
|
||||
if langfuse_handler:
|
||||
input_config["callbacks"] = [langfuse_handler]
|
||||
|
||||
async for msg, metadata in graph.astream(
|
||||
{"messages": messages},
|
||||
@ -125,9 +115,6 @@ class BaseAgent:
|
||||
"configurable": {"thread_id": context.thread_id, "user_id": context.user_id},
|
||||
"recursion_limit": 100,
|
||||
}
|
||||
langfuse_handler = get_langfuse_callback()
|
||||
if langfuse_handler:
|
||||
input_config["callbacks"] = [langfuse_handler]
|
||||
|
||||
msg = await graph.ainvoke(
|
||||
{"messages": messages},
|
||||
|
||||
@ -1,16 +1,11 @@
|
||||
import os
|
||||
import traceback
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_exponential
|
||||
|
||||
from src import config
|
||||
from src.utils import logger
|
||||
from src.utils.langfuse_integration import is_langfuse_enabled
|
||||
|
||||
if is_langfuse_enabled():
|
||||
from langfuse.openai import AsyncOpenAI
|
||||
else:
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
|
||||
def split_model_spec(model_spec, sep="/"):
|
||||
|
||||
@ -1,39 +0,0 @@
|
||||
"""
|
||||
Langfuse 集成模块
|
||||
|
||||
提供 Langfuse 的初始化和回调处理器获取功能。
|
||||
仅当环境变量 LANGFUSE_SECRET_KEY 和 LANGFUSE_PUBLIC_KEY 存在时启用。
|
||||
"""
|
||||
|
||||
import os
|
||||
from functools import lru_cache
|
||||
|
||||
from src.utils.logging_config import logger
|
||||
|
||||
|
||||
def is_langfuse_enabled() -> bool:
|
||||
"""检查 Langfuse 是否已配置"""
|
||||
return bool(
|
||||
os.getenv("LANGFUSE_SECRET_KEY")
|
||||
and os.getenv("LANGFUSE_PUBLIC_KEY")
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_langfuse_callback():
|
||||
"""获取 Langfuse LangChain CallbackHandler（单例）
|
||||
|
||||
Returns:
|
||||
CallbackHandler 实例，若 Langfuse 未配置则返回 None
|
||||
"""
|
||||
if not is_langfuse_enabled():
|
||||
return None
|
||||
try:
|
||||
from langfuse.langchain import CallbackHandler
|
||||
|
||||
handler = CallbackHandler()
|
||||
logger.info("Langfuse CallbackHandler 初始化成功")
|
||||
return handler
|
||||
except Exception as e:
|
||||
logger.warning(f"Langfuse CallbackHandler 初始化失败: {e}")
|
||||
return None
|
||||
@ -1,21 +0,0 @@
|
||||
"""GAIA 离线评估框架
|
||||
|
||||
使用 GAIA benchmark 评估 Yuxi-Know 的 Agent 系统。
|
||||
|
||||
用法:
|
||||
python -m test.gaia_eval --level 1 --max-samples 5
|
||||
"""
|
||||
|
||||
from .config import EvalConfig
|
||||
from .dataset_loader import GaiaDatasetLoader, GaiaTask
|
||||
from .runner import EvalResult, GaiaEvalRunner
|
||||
from .scorer import GaiaScorer
|
||||
|
||||
__all__ = [
|
||||
"EvalConfig",
|
||||
"GaiaDatasetLoader",
|
||||
"GaiaTask",
|
||||
"GaiaEvalRunner",
|
||||
"EvalResult",
|
||||
"GaiaScorer",
|
||||
]
|
||||
@ -1,109 +0,0 @@
|
||||
"""GAIA 离线评估 CLI 入口
|
||||
|
||||
用法:
|
||||
python -m test.gaia_eval # 评估全部级别
|
||||
python -m test.gaia_eval --level 1 # 只评估 Level 1
|
||||
python -m test.gaia_eval --level 1 --max-samples 5 # 限制 5 条（调试）
|
||||
python -m test.gaia_eval --agent-config-id 3 # 使用数据库预设配置
|
||||
python -m test.gaia_eval --model deepseek-chat --level 1 # 指定模型
|
||||
python -m test.gaia_eval --output-dir ./my_results # 自定义输出目录
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
from .config import EvalConfig
|
||||
from .dataset_loader import GaiaDatasetLoader
|
||||
from .reporter import GaiaReporter
|
||||
from .runner import GaiaEvalRunner
|
||||
|
||||
|
||||
def parse_args() -> EvalConfig:
|
||||
"""解析命令行参数"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="GAIA 离线评估 - 评估 Yuxi-Know Agent 系统",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
|
||||
parser.add_argument("--agent-id", default="ChatbotAgent", help="Agent ID（默认: ChatbotAgent）")
|
||||
parser.add_argument("--agent-config-id", type=int, default=None, help="Agent 预设配置 ID，从数据库加载")
|
||||
parser.add_argument("--model", default=None, help="模型名称（当未指定 agent-config-id 时使用）")
|
||||
parser.add_argument("--level", default="all", choices=["1", "2", "3", "all"], help="评估级别（默认: all）")
|
||||
parser.add_argument("--split", default="validation", choices=["validation", "test"], help="数据集划分（默认: validation）")
|
||||
parser.add_argument("--max-samples", type=int, default=None, help="最大评估样本数（调试用）")
|
||||
parser.add_argument("--timeout", type=int, default=300, help="单题超时，秒（默认: 300）")
|
||||
parser.add_argument("--concurrency", type=int, default=1, help="并发评估数（默认: 1）")
|
||||
parser.add_argument("--output-dir", default="eval_results", help="结果输出目录（默认: eval_results）")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
return EvalConfig(
|
||||
agent_id=args.agent_id,
|
||||
agent_config_id=args.agent_config_id,
|
||||
model=args.model,
|
||||
level=args.level,
|
||||
split=args.split,
|
||||
max_samples=args.max_samples,
|
||||
timeout=args.timeout,
|
||||
concurrency=args.concurrency,
|
||||
output_dir=Path(args.output_dir),
|
||||
)
|
||||
|
||||
|
||||
async def main():
|
||||
"""主流程"""
|
||||
from rich.console import Console
|
||||
|
||||
console = Console()
|
||||
config = parse_args()
|
||||
|
||||
# 1. 显示配置信息
|
||||
console.print("\n[bold cyan]═══ GAIA 离线评估 ═══[/bold cyan]")
|
||||
console.print(f" Agent: {config.agent_id}")
|
||||
if config.agent_config_id:
|
||||
console.print(f" 配置ID: {config.agent_config_id}")
|
||||
if config.model:
|
||||
console.print(f" 模型: {config.model}")
|
||||
console.print(f" 级别: {config.level}")
|
||||
console.print(f" 数据集: {config.split}")
|
||||
if config.max_samples:
|
||||
console.print(f" 样本限制: {config.max_samples}")
|
||||
console.print(f" 超时: {config.timeout}s")
|
||||
console.print(f" 并发: {config.concurrency}")
|
||||
console.print(f" 输出目录: {config.output_dir}")
|
||||
console.print()
|
||||
|
||||
# 2. 加载数据集
|
||||
console.print("[bold]正在加载 GAIA 数据集...[/bold]")
|
||||
try:
|
||||
loader = GaiaDatasetLoader(config)
|
||||
tasks = loader.load_tasks()
|
||||
except Exception as e:
|
||||
console.print(f"[red]数据集加载失败: {e}[/red]")
|
||||
console.print("[dim]请确保已设置 HF_TOKEN 并同意了数据集使用条款[/dim]")
|
||||
sys.exit(1)
|
||||
|
||||
console.print(f" 已加载 {len(tasks)} 条评估任务\n")
|
||||
|
||||
if not tasks:
|
||||
console.print("[yellow]没有可评估的任务[/yellow]")
|
||||
sys.exit(0)
|
||||
|
||||
# 3. 执行评估
|
||||
console.print("[bold]开始评估...[/bold]\n")
|
||||
runner = GaiaEvalRunner(config)
|
||||
results = await runner.run(tasks)
|
||||
|
||||
# 4. 生成报告
|
||||
reporter = GaiaReporter(config, results)
|
||||
reporter.print_report()
|
||||
|
||||
report_path = reporter.save_json_report()
|
||||
console.print(f"[green]详细报告已保存: {report_path}[/green]\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@ -1,57 +0,0 @@
|
||||
"""GAIA 评估配置管理模块"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvalConfig:
|
||||
"""GAIA 评估配置
|
||||
|
||||
配置优先级：CLI 参数 > 环境变量 > 默认值
|
||||
当指定 agent_config_id 时，会从数据库加载预设配置（model/tools/knowledges/system_prompt 等）
|
||||
"""
|
||||
|
||||
# Agent 相关
|
||||
agent_id: str = "ChatbotAgent"
|
||||
agent_config_id: int | None = None # 从数据库加载预设配置
|
||||
|
||||
# 备选配置（当未指定 agent_config_id 时使用）
|
||||
model: str | None = None
|
||||
system_prompt: str | None = None
|
||||
tools: list[dict] | None = None
|
||||
knowledges: list[str] | None = None
|
||||
|
||||
# 数据集相关
|
||||
level: str = "all" # "1", "2", "3", "all"
|
||||
split: str = "validation" # "validation" or "test"
|
||||
max_samples: int | None = None # 限制评估样本数（调试用）
|
||||
|
||||
# 运行控制
|
||||
timeout: int = 300 # 单题超时，秒
|
||||
concurrency: int = 1 # 并发评估数
|
||||
|
||||
# 输出相关
|
||||
output_dir: Path = field(default_factory=lambda: Path("eval_results"))
|
||||
|
||||
def get_dataset_configs(self) -> list[str]:
|
||||
"""根据 level 获取 HuggingFace dataset config 名称列表"""
|
||||
if self.level == "all":
|
||||
return ["2023_level1", "2023_level2", "2023_level3"]
|
||||
return [f"2023_level{self.level}"]
|
||||
|
||||
def build_agent_config(self) -> dict:
|
||||
"""构建不使用 agent_config_id 时的 agent_config 字典
|
||||
|
||||
当未指定 agent_config_id 时，从 CLI 参数组装配置。
|
||||
"""
|
||||
config = {}
|
||||
if self.model:
|
||||
config["model"] = self.model
|
||||
if self.system_prompt:
|
||||
config["system_prompt"] = self.system_prompt
|
||||
if self.tools is not None:
|
||||
config["tools"] = self.tools
|
||||
if self.knowledges is not None:
|
||||
config["knowledges"] = self.knowledges
|
||||
return config
|
||||
@ -1,164 +0,0 @@
|
||||
"""GAIA 数据集加载器
|
||||
|
||||
支持两种模式：
|
||||
1. 流式模式（默认）：直接 streaming 拉取 metadata，无需完整下载，适合快速测试
|
||||
2. 完整模式：snapshot_download 后加载，附件文件可本地读取
|
||||
|
||||
需要设置 HF_TOKEN 环境变量，并在 HuggingFace 上同意数据集使用条款。
|
||||
"""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .config import EvalConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class GaiaTask:
|
||||
"""单条 GAIA 评估任务"""
|
||||
|
||||
task_id: str
|
||||
question: str
|
||||
level: int
|
||||
final_answer: str
|
||||
file_name: str | None = None
|
||||
file_path: str | None = None # 流式模式下为 None（无本地文件）
|
||||
annotator_metadata: dict | None = None
|
||||
|
||||
|
||||
def _get_token() -> str | None:
|
||||
"""获取 HF_TOKEN，支持从环境变量或 .env 文件读取"""
|
||||
token = os.environ.get("HF_TOKEN")
|
||||
if not token:
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
token = os.environ.get("HF_TOKEN")
|
||||
except ImportError:
|
||||
pass
|
||||
return token
|
||||
|
||||
|
||||
class GaiaDatasetLoader:
|
||||
"""GAIA 数据集加载器"""
|
||||
|
||||
REPO_ID = "gaia-benchmark/GAIA"
|
||||
|
||||
def __init__(self, config: EvalConfig):
|
||||
self.config = config
|
||||
self._data_dir: str | None = None # 完整下载后的本地路径
|
||||
|
||||
def load_tasks(self) -> list[GaiaTask]:
|
||||
"""加载并返回 GaiaTask 列表
|
||||
|
||||
当 max_samples 较小时，自动使用流式加载避免完整下载；
|
||||
否则使用 snapshot_download 完整下载（支持附件文件读取）。
|
||||
"""
|
||||
# 小样本时用流式加载，避免下载完整数据集
|
||||
small_sample = self.config.max_samples and self.config.max_samples <= 50
|
||||
if small_sample:
|
||||
return self._load_streaming()
|
||||
else:
|
||||
return self._load_full()
|
||||
|
||||
def _load_streaming(self) -> list[GaiaTask]:
|
||||
"""流式加载：直接从 HuggingFace 流式拉取，附件文件按需单独下载"""
|
||||
from datasets import load_dataset
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
token = _get_token()
|
||||
if not token:
|
||||
raise RuntimeError(
|
||||
"未设置 HF_TOKEN 环境变量。请在 .env 中添加 HF_TOKEN=hf_xxx"
|
||||
)
|
||||
|
||||
dataset_configs = self.config.get_dataset_configs()
|
||||
tasks: list[GaiaTask] = []
|
||||
|
||||
for ds_config in dataset_configs:
|
||||
dataset = load_dataset(
|
||||
self.REPO_ID,
|
||||
ds_config,
|
||||
split=self.config.split,
|
||||
streaming=True,
|
||||
token=token,
|
||||
)
|
||||
|
||||
for example in dataset:
|
||||
file_name = example.get("file_name") or None
|
||||
file_path = None
|
||||
|
||||
# 按需下载单个附件文件（只下载当前任务的文件，不下载整个数据集）
|
||||
if file_name:
|
||||
repo_file_path = f"2023/{self.config.split}/{file_name}"
|
||||
try:
|
||||
file_path = hf_hub_download(
|
||||
repo_id=self.REPO_ID,
|
||||
repo_type="dataset",
|
||||
filename=repo_file_path,
|
||||
token=token,
|
||||
)
|
||||
except Exception as e:
|
||||
import warnings
|
||||
warnings.warn(f"附件下载失败 {file_name}: {e}")
|
||||
|
||||
task = GaiaTask(
|
||||
task_id=example["task_id"],
|
||||
question=example["Question"],
|
||||
level=int(example["Level"]),
|
||||
final_answer=example.get("Final answer", ""),
|
||||
file_name=file_name,
|
||||
file_path=file_path,
|
||||
annotator_metadata=example.get("Annotator Metadata"),
|
||||
)
|
||||
tasks.append(task)
|
||||
|
||||
if self.config.max_samples and len(tasks) >= self.config.max_samples:
|
||||
return tasks
|
||||
|
||||
return tasks
|
||||
|
||||
def _load_full(self) -> list[GaiaTask]:
|
||||
"""完整下载：先 snapshot_download，再加载（附件文件可本地读取）"""
|
||||
from datasets import load_dataset
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
token = _get_token()
|
||||
if not token:
|
||||
raise RuntimeError(
|
||||
"未设置 HF_TOKEN 环境变量。请在 .env 中添加 HF_TOKEN=hf_xxx"
|
||||
)
|
||||
|
||||
if not self._data_dir:
|
||||
self._data_dir = snapshot_download(
|
||||
repo_id=self.REPO_ID,
|
||||
repo_type="dataset",
|
||||
token=token,
|
||||
)
|
||||
|
||||
dataset_configs = self.config.get_dataset_configs()
|
||||
tasks: list[GaiaTask] = []
|
||||
|
||||
for ds_config in dataset_configs:
|
||||
dataset = load_dataset(self._data_dir, ds_config, split=self.config.split)
|
||||
|
||||
for example in dataset:
|
||||
file_path = example.get("file_path")
|
||||
if file_path:
|
||||
file_path = os.path.join(self._data_dir, file_path)
|
||||
|
||||
task = GaiaTask(
|
||||
task_id=example["task_id"],
|
||||
question=example["Question"],
|
||||
level=int(example["Level"]),
|
||||
final_answer=example.get("Final answer", ""),
|
||||
file_name=example.get("file_name") or None,
|
||||
file_path=file_path,
|
||||
annotator_metadata=example.get("Annotator Metadata"),
|
||||
)
|
||||
tasks.append(task)
|
||||
|
||||
if self.config.max_samples and len(tasks) > self.config.max_samples:
|
||||
tasks = tasks[: self.config.max_samples]
|
||||
|
||||
return tasks
|
||||
@ -1,191 +0,0 @@
|
||||
"""GAIA 评估文件处理模块
|
||||
|
||||
将 GAIA 数据集中的附件文件转换为 Agent 可识别的 attachments + state files 格式。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import mimetypes
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
from src.utils import logger
|
||||
|
||||
|
||||
# 支持直接读取内容的文本类文件扩展名
|
||||
TEXT_EXTENSIONS = {
|
||||
".txt", ".md", ".csv", ".tsv", ".json", ".jsonl",
|
||||
".xml", ".html", ".htm", ".yaml", ".yml", ".log",
|
||||
".py", ".js", ".ts", ".java", ".c", ".cpp", ".h",
|
||||
".rb", ".go", ".rs", ".sh", ".bat", ".sql",
|
||||
".ini", ".cfg", ".conf", ".toml",
|
||||
}
|
||||
|
||||
# 支持用 PDF 解析的扩展名
|
||||
PDF_EXTENSIONS = {".pdf"}
|
||||
|
||||
# 支持用 pandas 等读取的表格扩展名
|
||||
SPREADSHEET_EXTENSIONS = {".xlsx", ".xls"}
|
||||
|
||||
# 图片类扩展名（用于多模态消息）
|
||||
IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp"}
|
||||
|
||||
|
||||
def _read_text_file(file_path: str) -> str | None:
|
||||
"""读取纯文本文件"""
|
||||
encodings = ["utf-8", "latin-1", "gbk"]
|
||||
for enc in encodings:
|
||||
try:
|
||||
with open(file_path, encoding=enc) as f:
|
||||
return f.read()
|
||||
except (UnicodeDecodeError, OSError):
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _read_pdf_file(file_path: str) -> str | None:
|
||||
"""解析 PDF 文件为文本"""
|
||||
try:
|
||||
import fitz # PyMuPDF
|
||||
|
||||
doc = fitz.open(file_path)
|
||||
pages = []
|
||||
for page in doc:
|
||||
pages.append(page.get_text())
|
||||
doc.close()
|
||||
return "\n\n".join(pages)
|
||||
except ImportError:
|
||||
logger.warning("PyMuPDF (fitz) 未安装，无法解析 PDF。请安装: pip install PyMuPDF")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning(f"PDF 解析失败 {file_path}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def _read_spreadsheet_file(file_path: str) -> str | None:
|
||||
"""读取 Excel 文件并转为 CSV 文本"""
|
||||
try:
|
||||
import pandas as pd
|
||||
|
||||
df = pd.read_excel(file_path)
|
||||
return df.to_csv(index=False)
|
||||
except ImportError:
|
||||
logger.warning("pandas/openpyxl 未安装，无法解析 Excel。请安装: pip install pandas openpyxl")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning(f"Excel 解析失败 {file_path}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def _read_image_as_base64(file_path: str) -> str | None:
|
||||
"""读取图片为 base64"""
|
||||
try:
|
||||
with open(file_path, "rb") as f:
|
||||
data = f.read()
|
||||
return base64.b64encode(data).decode("utf-8")
|
||||
except Exception as e:
|
||||
logger.warning(f"图片读取失败 {file_path}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def read_file_content(file_path: str) -> str | None:
|
||||
"""根据文件类型读取内容，返回文本"""
|
||||
if not file_path or not os.path.exists(file_path):
|
||||
logger.warning(f"文件不存在: {file_path}")
|
||||
return None
|
||||
|
||||
_, ext = os.path.splitext(file_path)
|
||||
ext = ext.lower()
|
||||
|
||||
if ext in TEXT_EXTENSIONS:
|
||||
return _read_text_file(file_path)
|
||||
elif ext in PDF_EXTENSIONS:
|
||||
return _read_pdf_file(file_path)
|
||||
elif ext in SPREADSHEET_EXTENSIONS:
|
||||
return _read_spreadsheet_file(file_path)
|
||||
else:
|
||||
# 尝试作为文本读取
|
||||
content = _read_text_file(file_path)
|
||||
if content:
|
||||
return content
|
||||
logger.warning(f"不支持的文件类型: {ext} ({file_path})")
|
||||
return None
|
||||
|
||||
|
||||
def build_attachments_and_files(
|
||||
file_path: str,
|
||||
file_name: str | None = None,
|
||||
) -> tuple[list[dict], dict]:
|
||||
"""将文件转换为 Agent 的 attachments 和 state files 格式
|
||||
|
||||
返回:
|
||||
(attachments, files) 元组：
|
||||
- attachments: 附件元数据列表，用于 AttachmentMiddleware
|
||||
- files: StateBackend 格式的文件字典，用于 read_file 工具
|
||||
"""
|
||||
if not file_path or not os.path.exists(file_path):
|
||||
return [], {}
|
||||
|
||||
if not file_name:
|
||||
file_name = os.path.basename(file_path)
|
||||
|
||||
_, ext = os.path.splitext(file_path)
|
||||
ext = ext.lower()
|
||||
|
||||
# 图片使用多模态消息，不走 attachment 机制
|
||||
if ext in IMAGE_EXTENSIONS:
|
||||
return [], {}
|
||||
|
||||
content = read_file_content(file_path)
|
||||
if not content:
|
||||
return [], {}
|
||||
|
||||
now = datetime.utcnow().isoformat() + "+00:00"
|
||||
virtual_path = f"/attachments/{file_name}"
|
||||
|
||||
# 构建 attachment 元数据
|
||||
attachment = {
|
||||
"file_name": file_name,
|
||||
"file_path": virtual_path,
|
||||
"status": "parsed",
|
||||
"markdown": content,
|
||||
"uploaded_at": now,
|
||||
"truncated": False,
|
||||
}
|
||||
|
||||
# 构建 state files（与 _build_state_files 格式一致）
|
||||
content_lines = content.split("\n")
|
||||
files = {
|
||||
virtual_path: {
|
||||
"content": content_lines,
|
||||
"created_at": now,
|
||||
"modified_at": now,
|
||||
}
|
||||
}
|
||||
|
||||
return [attachment], files
|
||||
|
||||
|
||||
def build_image_message_content(file_path: str, question: str) -> list[dict] | None:
|
||||
"""为图片文件构建多模态消息内容
|
||||
|
||||
返回 HumanMessage 的 content（多模态格式），或 None 如果不是图片。
|
||||
"""
|
||||
if not file_path or not os.path.exists(file_path):
|
||||
return None
|
||||
|
||||
_, ext = os.path.splitext(file_path)
|
||||
if ext.lower() not in IMAGE_EXTENSIONS:
|
||||
return None
|
||||
|
||||
base64_data = _read_image_as_base64(file_path)
|
||||
if not base64_data:
|
||||
return None
|
||||
|
||||
mime_type = mimetypes.guess_type(file_path)[0] or "image/jpeg"
|
||||
|
||||
return [
|
||||
{"type": "text", "text": question},
|
||||
{"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{base64_data}"}},
|
||||
]
|
||||
@ -1,162 +0,0 @@
|
||||
"""GAIA 评估报告生成模块
|
||||
|
||||
输出终端彩色表格和 JSON 详细报告。
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from .config import EvalConfig
|
||||
from .runner import EvalResult
|
||||
|
||||
|
||||
class GaiaReporter:
|
||||
"""评估报告生成器"""
|
||||
|
||||
def __init__(self, config: EvalConfig, results: list[EvalResult]):
|
||||
self.config = config
|
||||
self.results = results
|
||||
|
||||
def _compute_stats(self) -> dict:
|
||||
"""计算统计数据"""
|
||||
total = len(self.results)
|
||||
correct = sum(1 for r in self.results if r.is_correct)
|
||||
errors = sum(1 for r in self.results if r.error is not None)
|
||||
|
||||
# 按 Level 分组统计
|
||||
by_level: dict[int, dict] = defaultdict(lambda: {"total": 0, "correct": 0, "errors": 0})
|
||||
for r in self.results:
|
||||
by_level[r.level]["total"] += 1
|
||||
if r.is_correct:
|
||||
by_level[r.level]["correct"] += 1
|
||||
if r.error:
|
||||
by_level[r.level]["errors"] += 1
|
||||
|
||||
# 耗时统计
|
||||
durations = [r.duration_seconds for r in self.results if not r.error]
|
||||
avg_duration = sum(durations) / len(durations) if durations else 0
|
||||
max_duration = max(durations) if durations else 0
|
||||
min_duration = min(durations) if durations else 0
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"correct": correct,
|
||||
"accuracy": round(correct / total * 100, 2) if total > 0 else 0,
|
||||
"errors": errors,
|
||||
"by_level": {
|
||||
level: {
|
||||
"total": stats["total"],
|
||||
"correct": stats["correct"],
|
||||
"accuracy": round(stats["correct"] / stats["total"] * 100, 2) if stats["total"] > 0 else 0,
|
||||
"errors": stats["errors"],
|
||||
}
|
||||
for level, stats in sorted(by_level.items())
|
||||
},
|
||||
"duration": {
|
||||
"avg": round(avg_duration, 2),
|
||||
"max": round(max_duration, 2),
|
||||
"min": round(min_duration, 2),
|
||||
"total": round(sum(durations), 2),
|
||||
},
|
||||
}
|
||||
|
||||
def print_report(self):
|
||||
"""在终端输出彩色报告"""
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
console = Console()
|
||||
stats = self._compute_stats()
|
||||
|
||||
# 标题
|
||||
console.print("\n[bold cyan]═══ GAIA 评估报告 ═══[/bold cyan]\n")
|
||||
|
||||
# 总体统计
|
||||
accuracy_color = "green" if stats["accuracy"] >= 50 else ("yellow" if stats["accuracy"] >= 25 else "red")
|
||||
console.print(f" Agent: [bold]{self.config.agent_id}[/bold]")
|
||||
if self.config.agent_config_id:
|
||||
console.print(f" 配置ID: [bold]{self.config.agent_config_id}[/bold]")
|
||||
console.print(f" 总题数: {stats['total']}")
|
||||
console.print(f" 正确数: {stats['correct']}")
|
||||
console.print(f" 错误数: {stats['errors']}")
|
||||
console.print(f" 准确率: [{accuracy_color}]{stats['accuracy']}%[/{accuracy_color}]")
|
||||
console.print()
|
||||
|
||||
# 按 Level 统计表
|
||||
level_table = Table(title="按 Level 统计")
|
||||
level_table.add_column("Level", style="bold")
|
||||
level_table.add_column("总数", justify="right")
|
||||
level_table.add_column("正确", justify="right")
|
||||
level_table.add_column("准确率", justify="right")
|
||||
level_table.add_column("错误", justify="right")
|
||||
|
||||
for level, level_stats in stats["by_level"].items():
|
||||
acc_color = "green" if level_stats["accuracy"] >= 50 else (
|
||||
"yellow" if level_stats["accuracy"] >= 25 else "red"
|
||||
)
|
||||
level_table.add_row(
|
||||
f"Level {level}",
|
||||
str(level_stats["total"]),
|
||||
str(level_stats["correct"]),
|
||||
f"[{acc_color}]{level_stats['accuracy']}%[/{acc_color}]",
|
||||
str(level_stats["errors"]),
|
||||
)
|
||||
|
||||
console.print(level_table)
|
||||
console.print()
|
||||
|
||||
# 耗时统计
|
||||
console.print("[bold]耗时统计:[/bold]")
|
||||
console.print(f" 平均: {stats['duration']['avg']}s")
|
||||
console.print(f" 最大: {stats['duration']['max']}s")
|
||||
console.print(f" 最小: {stats['duration']['min']}s")
|
||||
console.print(f" 总计: {stats['duration']['total']}s")
|
||||
console.print()
|
||||
|
||||
# 错误样本
|
||||
error_results = [r for r in self.results if r.error]
|
||||
if error_results:
|
||||
console.print(f"[bold red]错误样本 ({len(error_results)} 条):[/bold red]")
|
||||
for r in error_results[:5]:
|
||||
console.print(f" • [{r.task_id[:8]}] L{r.level}: {r.error}")
|
||||
if len(error_results) > 5:
|
||||
console.print(f" ... 还有 {len(error_results) - 5} 条错误")
|
||||
console.print()
|
||||
|
||||
def save_json_report(self) -> Path:
|
||||
"""保存 JSON 详细报告"""
|
||||
self.config.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
stats = self._compute_stats()
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
report_path = self.config.output_dir / f"gaia_eval_{timestamp}.json"
|
||||
|
||||
report = {
|
||||
"metadata": {
|
||||
"timestamp": timestamp,
|
||||
"agent_id": self.config.agent_id,
|
||||
"agent_config_id": self.config.agent_config_id,
|
||||
"level": self.config.level,
|
||||
"split": self.config.split,
|
||||
"timeout": self.config.timeout,
|
||||
},
|
||||
"summary": stats,
|
||||
"results": [
|
||||
{
|
||||
"task_id": r.task_id,
|
||||
"level": r.level,
|
||||
"question": r.question,
|
||||
"gold_answer": r.gold_answer,
|
||||
"predicted_answer": r.predicted_answer,
|
||||
"is_correct": r.is_correct,
|
||||
"error": r.error,
|
||||
"duration_seconds": r.duration_seconds,
|
||||
}
|
||||
for r in self.results
|
||||
],
|
||||
}
|
||||
|
||||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return report_path
|
||||
@ -1,272 +0,0 @@
|
||||
"""GAIA 评估运行器
|
||||
|
||||
核心评估逻辑：加载 agent、执行评估、收集结果。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
from .config import EvalConfig
|
||||
from .dataset_loader import GaiaTask
|
||||
from .scorer import GaiaScorer
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvalResult:
|
||||
"""单条评估结果"""
|
||||
|
||||
task_id: str
|
||||
level: int
|
||||
question: str
|
||||
gold_answer: str
|
||||
predicted_answer: str
|
||||
is_correct: bool
|
||||
error: str | None = None
|
||||
duration_seconds: float = 0.0
|
||||
|
||||
|
||||
class GaiaEvalRunner:
|
||||
"""GAIA 评估运行器
|
||||
|
||||
通过直接调用 BaseAgent.invoke_messages() 执行评估，
|
||||
无需启动 Web 服务。
|
||||
"""
|
||||
|
||||
def __init__(self, config: EvalConfig):
|
||||
self.config = config
|
||||
self._agent = None
|
||||
self._agent_config: dict = {}
|
||||
|
||||
async def _init_agent(self):
|
||||
"""初始化 Agent 实例并加载预设配置"""
|
||||
from src.agents import agent_manager
|
||||
|
||||
self._agent = agent_manager.get_agent(self.config.agent_id)
|
||||
|
||||
# 如果指定了 agent_config_id，从数据库加载预设配置
|
||||
if self.config.agent_config_id is not None:
|
||||
await self._load_agent_config_from_db()
|
||||
else:
|
||||
# 使用 CLI 参数组装配置
|
||||
self._agent_config = self.config.build_agent_config()
|
||||
|
||||
async def _load_agent_config_from_db(self):
|
||||
"""从数据库加载 agent 预设配置"""
|
||||
from src.repositories.agent_config_repository import AgentConfigRepository
|
||||
from src.storage.postgres.manager import pg_manager
|
||||
|
||||
# 评估脚本独立运行时，pg_manager 和 mcp_service 未经过 FastAPI 启动事件初始化，需手动初始化
|
||||
if not pg_manager._initialized:
|
||||
pg_manager.initialize()
|
||||
|
||||
from src.services.mcp_service import init_mcp_servers
|
||||
await init_mcp_servers()
|
||||
|
||||
async with pg_manager.get_async_session_context() as db:
|
||||
repo = AgentConfigRepository(db)
|
||||
config_item = await repo.get_by_id(self.config.agent_config_id)
|
||||
|
||||
if config_item is None:
|
||||
raise ValueError(f"Agent 配置 ID {self.config.agent_config_id} 不存在")
|
||||
|
||||
# 与线上逻辑保持一致：从 config_json.context 中提取配置
|
||||
self._agent_config = (config_item.config_json or {}).get("context", {})
|
||||
|
||||
def _extract_answer(self, result: dict) -> str:
|
||||
"""从 agent 返回结果中提取最终答案
|
||||
|
||||
Agent 返回格式为 { "messages": [...] }，取最后一条 AI 消息的内容。
|
||||
"""
|
||||
messages = result.get("messages", [])
|
||||
if not messages:
|
||||
return ""
|
||||
|
||||
# 从后往前找第一条 AI 消息
|
||||
for msg in reversed(messages):
|
||||
msg_type = getattr(msg, "type", None)
|
||||
if msg_type == "ai":
|
||||
content = getattr(msg, "content", "")
|
||||
if isinstance(content, str) and content.strip():
|
||||
return self._parse_final_answer(content)
|
||||
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _parse_final_answer(content: str) -> str:
|
||||
"""尝试从 AI 回复中解析出最终答案
|
||||
|
||||
如果回复中包含 "FINAL ANSWER:" 标记，则提取标记后面的内容。
|
||||
否则返回完整回复。
|
||||
"""
|
||||
# 常见答案标记格式
|
||||
markers = [
|
||||
"FINAL ANSWER:",
|
||||
"Final Answer:",
|
||||
"final answer:",
|
||||
"最终答案：",
|
||||
"最终答案:",
|
||||
"答案：",
|
||||
"答案:",
|
||||
]
|
||||
for marker in markers:
|
||||
if marker in content:
|
||||
answer = content.split(marker, 1)[1].strip()
|
||||
# 取第一行（去掉后续的解释）
|
||||
first_line = answer.split("\n")[0].strip()
|
||||
return first_line
|
||||
|
||||
# 如果只有一行，直接返回
|
||||
lines = [ln.strip() for ln in content.strip().split("\n") if ln.strip()]
|
||||
if len(lines) == 1:
|
||||
return lines[0]
|
||||
|
||||
# 多行回复，返回最后一行（通常是最终答案）
|
||||
return lines[-1] if lines else content.strip()
|
||||
|
||||
async def _run_single_task(self, task: GaiaTask) -> EvalResult:
|
||||
"""执行单条评估任务"""
|
||||
start_time = time.time()
|
||||
predicted_answer = ""
|
||||
error = None
|
||||
|
||||
try:
|
||||
from langchain.messages import HumanMessage
|
||||
|
||||
from .file_handler import build_attachments_and_files, build_image_message_content
|
||||
|
||||
# 构建系统提示词
|
||||
gaia_system_suffix = (
|
||||
"\n\nIMPORTANT: When you have found the answer, "
|
||||
"state it clearly on a single line prefixed with 'FINAL ANSWER: '. "
|
||||
"Keep the answer as concise as possible - just the answer itself, "
|
||||
"no extra explanation."
|
||||
)
|
||||
|
||||
input_context = {
|
||||
"user_id": "gaia_eval",
|
||||
"thread_id": str(uuid.uuid4()),
|
||||
"agent_config": {
|
||||
**self._agent_config,
|
||||
"system_prompt": (
|
||||
self._agent_config.get("system_prompt", "You are a helpful assistant.")
|
||||
+ gaia_system_suffix
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
# 处理附件文件
|
||||
attachments: list[dict] = []
|
||||
files: dict = {}
|
||||
image_content: list[dict] | None = None
|
||||
|
||||
if task.file_path:
|
||||
# 尝试构建图片多模态消息
|
||||
image_content = build_image_message_content(task.file_path, task.question)
|
||||
|
||||
if not image_content:
|
||||
# 非图片文件：通过 attachments + files 注入 state
|
||||
attachments, files = build_attachments_and_files(
|
||||
task.file_path, task.file_name
|
||||
)
|
||||
|
||||
# 构建消息
|
||||
if image_content:
|
||||
# 图片：使用多模态 HumanMessage
|
||||
messages = [HumanMessage(content=image_content)]
|
||||
else:
|
||||
messages = [HumanMessage(content=task.question)]
|
||||
|
||||
# 构建 graph 输入（包含 attachments 和 files）
|
||||
graph_input = {"messages": messages}
|
||||
if attachments:
|
||||
graph_input["attachments"] = attachments
|
||||
if files:
|
||||
graph_input["files"] = files
|
||||
|
||||
# 直接调用 graph.ainvoke 以传递完整的初始 state
|
||||
graph = await self._agent.get_graph()
|
||||
context = self._agent.context_schema()
|
||||
agent_config = input_context.get("agent_config")
|
||||
if isinstance(agent_config, dict):
|
||||
context.update(agent_config)
|
||||
context.update(input_context)
|
||||
|
||||
invoke_config = {
|
||||
"configurable": {
|
||||
"thread_id": context.thread_id,
|
||||
"user_id": context.user_id,
|
||||
},
|
||||
"recursion_limit": 100,
|
||||
}
|
||||
|
||||
# 设置超时
|
||||
result = await asyncio.wait_for(
|
||||
graph.ainvoke(graph_input, context=context, config=invoke_config),
|
||||
timeout=self.config.timeout,
|
||||
)
|
||||
|
||||
predicted_answer = self._extract_answer(result)
|
||||
|
||||
except TimeoutError:
|
||||
error = f"Timeout after {self.config.timeout}s"
|
||||
except Exception as e:
|
||||
error = f"{type(e).__name__}: {e}"
|
||||
|
||||
duration = time.time() - start_time
|
||||
is_correct = GaiaScorer.score(predicted_answer, task.final_answer) if not error else False
|
||||
|
||||
return EvalResult(
|
||||
task_id=task.task_id,
|
||||
level=task.level,
|
||||
question=task.question,
|
||||
gold_answer=task.final_answer,
|
||||
predicted_answer=predicted_answer,
|
||||
is_correct=is_correct,
|
||||
error=error,
|
||||
duration_seconds=round(duration, 2),
|
||||
)
|
||||
|
||||
async def run(self, tasks: list[GaiaTask]) -> list[EvalResult]:
|
||||
"""执行批量评估
|
||||
|
||||
Args:
|
||||
tasks: 待评估的 GaiaTask 列表
|
||||
|
||||
Returns:
|
||||
评估结果列表
|
||||
"""
|
||||
await self._init_agent()
|
||||
|
||||
results: list[EvalResult] = []
|
||||
|
||||
if self.config.concurrency <= 1:
|
||||
# 串行执行
|
||||
for task in tqdm(tasks, desc="评估进度", unit="题"):
|
||||
result = await self._run_single_task(task)
|
||||
results.append(result)
|
||||
status = "✓" if result.is_correct else ("✗" if not result.error else "⚠")
|
||||
tqdm.write(
|
||||
f" {status} [{result.task_id[:8]}] L{result.level} "
|
||||
f"| 预测: {result.predicted_answer[:50]!r} "
|
||||
f"| 标准: {result.gold_answer[:50]!r} "
|
||||
f"| {result.duration_seconds:.1f}s"
|
||||
)
|
||||
else:
|
||||
# 并发执行（使用信号量控制并发度）
|
||||
semaphore = asyncio.Semaphore(self.config.concurrency)
|
||||
pbar = tqdm(total=len(tasks), desc="评估进度", unit="题")
|
||||
|
||||
async def bounded_run(task: GaiaTask) -> EvalResult:
|
||||
async with semaphore:
|
||||
result = await self._run_single_task(task)
|
||||
pbar.update(1)
|
||||
return result
|
||||
|
||||
results = await asyncio.gather(*[bounded_run(t) for t in tasks])
|
||||
pbar.close()
|
||||
|
||||
return results
|
||||
@ -1,97 +0,0 @@
|
||||
"""GAIA 答案评分模块
|
||||
|
||||
实现 GAIA 官方的 quasi exact match 评分逻辑。
|
||||
参考: https://huggingface.co/spaces/gaia-benchmark/leaderboard
|
||||
"""
|
||||
|
||||
import re
|
||||
import string
|
||||
|
||||
|
||||
class GaiaScorer:
|
||||
"""GAIA 准精确匹配评分器"""
|
||||
|
||||
@staticmethod
|
||||
def normalize_text(text: str) -> str:
|
||||
"""文本标准化
|
||||
|
||||
- 去除首尾空白
|
||||
- 统一为小写
|
||||
- 移除冠词 (a, an, the)
|
||||
- 移除标点
|
||||
- 合并连续空白
|
||||
"""
|
||||
text = text.strip().lower()
|
||||
|
||||
# 移除冠词
|
||||
text = re.sub(r"\b(a|an|the)\b", " ", text)
|
||||
|
||||
# 移除标点
|
||||
text = text.translate(str.maketrans("", "", string.punctuation))
|
||||
|
||||
# 合并连续空白
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
return text
|
||||
|
||||
@staticmethod
|
||||
def normalize_number(text: str) -> str:
|
||||
"""数字标准化
|
||||
|
||||
- "1,000" → "1000"
|
||||
- "3.0" → "3"
|
||||
- "$100" → "100"
|
||||
- "100%" → "100"
|
||||
"""
|
||||
# 去除货币符号
|
||||
text = re.sub(r"[$€£¥]", "", text)
|
||||
# 去除百分号
|
||||
text = text.rstrip("%")
|
||||
# 去除千分位逗号
|
||||
text = text.replace(",", "")
|
||||
|
||||
# 判断是否为数字，若是则标准化
|
||||
try:
|
||||
num = float(text)
|
||||
# 如果是整数则去掉 .0
|
||||
if num == int(num):
|
||||
return str(int(num))
|
||||
return str(num)
|
||||
except ValueError:
|
||||
return text
|
||||
|
||||
@classmethod
|
||||
def score(cls, prediction: str, gold: str) -> bool:
|
||||
"""计算单条评估的准精确匹配分数
|
||||
|
||||
Args:
|
||||
prediction: 模型预测答案
|
||||
gold: 黄金标准答案
|
||||
|
||||
Returns:
|
||||
True 表示匹配，False 表示不匹配
|
||||
"""
|
||||
if not prediction or not gold:
|
||||
return False
|
||||
|
||||
# 先尝试数字比较
|
||||
pred_num = cls.normalize_number(prediction.strip())
|
||||
gold_num = cls.normalize_number(gold.strip())
|
||||
try:
|
||||
if float(pred_num) == float(gold_num):
|
||||
return True
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# 检查是否为列表答案（逗号分隔）
|
||||
if "," in gold:
|
||||
pred_items = sorted(cls.normalize_text(item) for item in prediction.split(","))
|
||||
gold_items = sorted(cls.normalize_text(item) for item in gold.split(","))
|
||||
if pred_items == gold_items:
|
||||
return True
|
||||
|
||||
# 文本标准化比较
|
||||
pred_normalized = cls.normalize_text(prediction)
|
||||
gold_normalized = cls.normalize_text(gold)
|
||||
|
||||
return pred_normalized == gold_normalized
|
||||
@ -1,102 +0,0 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# 确保可以找到 gaia_eval 包
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
from gaia_eval.scorer import GaiaScorer
|
||||
|
||||
|
||||
|
||||
class TestNormalizeText:
|
||||
"""文本标准化测试"""
|
||||
|
||||
def test_basic(self):
|
||||
assert GaiaScorer.normalize_text(" Hello World ") == "hello world"
|
||||
|
||||
def test_remove_articles(self):
|
||||
assert GaiaScorer.normalize_text("The quick brown fox") == "quick brown fox"
|
||||
assert GaiaScorer.normalize_text("a cat") == "cat"
|
||||
assert GaiaScorer.normalize_text("an apple") == "apple"
|
||||
|
||||
def test_remove_punctuation(self):
|
||||
assert GaiaScorer.normalize_text("Hello, World!") == "hello world"
|
||||
assert GaiaScorer.normalize_text("Dr. Smith's") == "dr smiths"
|
||||
|
||||
def test_collapse_whitespace(self):
|
||||
assert GaiaScorer.normalize_text("hello world") == "hello world"
|
||||
|
||||
|
||||
class TestNormalizeNumber:
|
||||
"""数字标准化测试"""
|
||||
|
||||
def test_remove_commas(self):
|
||||
assert GaiaScorer.normalize_number("1,000") == "1000"
|
||||
assert GaiaScorer.normalize_number("1,234,567") == "1234567"
|
||||
|
||||
def test_remove_trailing_zeros(self):
|
||||
assert GaiaScorer.normalize_number("3.0") == "3"
|
||||
assert GaiaScorer.normalize_number("100.00") == "100"
|
||||
|
||||
def test_keep_decimals(self):
|
||||
assert GaiaScorer.normalize_number("3.14") == "3.14"
|
||||
|
||||
def test_currency(self):
|
||||
assert GaiaScorer.normalize_number("$100") == "100"
|
||||
assert GaiaScorer.normalize_number("€50") == "50"
|
||||
|
||||
def test_percentage(self):
|
||||
assert GaiaScorer.normalize_number("75%") == "75"
|
||||
|
||||
def test_non_number(self):
|
||||
assert GaiaScorer.normalize_number("hello") == "hello"
|
||||
|
||||
|
||||
class TestScore:
|
||||
"""评分逻辑测试"""
|
||||
|
||||
def test_exact_match(self):
|
||||
assert GaiaScorer.score("Paris", "Paris") is True
|
||||
|
||||
def test_case_insensitive(self):
|
||||
assert GaiaScorer.score("paris", "Paris") is True
|
||||
assert GaiaScorer.score("PARIS", "paris") is True
|
||||
|
||||
def test_with_articles(self):
|
||||
assert GaiaScorer.score("The Eiffel Tower", "Eiffel Tower") is True
|
||||
|
||||
def test_number_match(self):
|
||||
assert GaiaScorer.score("1,000", "1000") is True
|
||||
assert GaiaScorer.score("3.0", "3") is True
|
||||
assert GaiaScorer.score("$100", "100") is True
|
||||
|
||||
def test_list_match(self):
|
||||
assert GaiaScorer.score("a, b, c", "c, b, a") is True
|
||||
assert GaiaScorer.score("Apple, Banana", "Banana, Apple") is True
|
||||
|
||||
def test_list_mismatch(self):
|
||||
assert GaiaScorer.score("a, b", "a, b, c") is False
|
||||
|
||||
def test_mismatch(self):
|
||||
assert GaiaScorer.score("London", "Paris") is False
|
||||
|
||||
def test_empty_prediction(self):
|
||||
assert GaiaScorer.score("", "Paris") is False
|
||||
|
||||
def test_empty_gold(self):
|
||||
assert GaiaScorer.score("Paris", "") is False
|
||||
|
||||
def test_both_empty(self):
|
||||
assert GaiaScorer.score("", "") is False
|
||||
|
||||
def test_whitespace_handling(self):
|
||||
assert GaiaScorer.score(" Paris ", "Paris") is True
|
||||
|
||||
def test_number_with_text(self):
|
||||
# 混合的数字和文本不应该触发数字匹配
|
||||
assert GaiaScorer.score("42 years", "42") is False
|
||||
|
||||
def test_floating_point(self):
|
||||
assert GaiaScorer.score("3.14", "3.14") is True
|
||||
assert GaiaScorer.score("3.14", "3.15") is False
|
||||
90
uv.lock
90
uv.lock
@ -611,31 +611,6 @@ wheels = [
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686, upload-time = "2024-06-09T16:20:16.715Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "datasets"
|
||||
version = "4.5.0"
|
||||
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
|
||||
dependencies = [
|
||||
{ name = "dill" },
|
||||
{ name = "filelock" },
|
||||
{ name = "fsspec", extra = ["http"] },
|
||||
{ name = "httpx" },
|
||||
{ name = "huggingface-hub" },
|
||||
{ name = "multiprocess" },
|
||||
{ name = "numpy" },
|
||||
{ name = "packaging" },
|
||||
{ name = "pandas" },
|
||||
{ name = "pyarrow" },
|
||||
{ name = "pyyaml" },
|
||||
{ name = "requests" },
|
||||
{ name = "tqdm" },
|
||||
{ name = "xxhash" },
|
||||
]
|
||||
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/55/bf/bb927bde63d649296c83e883171ae77074717c1b80fe2868b328bd0dbcbb/datasets-4.5.0.tar.gz", hash = "sha256:00c698ce1c2452e646cc5fad47fef39d3fe78dd650a8a6eb205bb45eb63cd500", size = 588384, upload-time = "2026-01-14T18:27:54.297Z" }
|
||||
wheels = [
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/fc/d5/0d563ea3c205eee226dc8053cf7682a8ac588db8acecd0eda2b587987a0b/datasets-4.5.0-py3-none-any.whl", hash = "sha256:b5d7e08096ffa407dd69e58b1c0271c9b2506140839b8d99af07375ad31b6726", size = 515196, upload-time = "2026-01-14T18:27:52.419Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deepagents"
|
||||
version = "0.3.6"
|
||||
@ -993,11 +968,6 @@ wheels = [
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/eb/02/a6b21098b1d5d6249b7c5ab69dde30108a71e4e819d4a9778f1de1d5b70d/fsspec-2025.10.0-py3-none-any.whl", hash = "sha256:7c7712353ae7d875407f97715f0e1ffcc21e33d5b24556cb1e090ae9409ec61d", size = 200966, upload-time = "2025-10-30T14:58:42.53Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
http = [
|
||||
{ name = "aiohttp" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "google-api-core"
|
||||
version = "2.29.0"
|
||||
@ -1720,27 +1690,6 @@ dependencies = [
|
||||
]
|
||||
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0e/72/a3add0e4eec4eb9e2569554f7c70f4a3c27712f40e3284d483e88094cc0e/langdetect-1.0.9.tar.gz", hash = "sha256:cbc1fef89f8d062739774bd51eda3da3274006b3661d199c2655f6b3f6d605a0", size = 981474, upload-time = "2021-05-07T07:54:13.562Z" }
|
||||
|
||||
[[package]]
|
||||
name = "langfuse"
|
||||
version = "3.14.5"
|
||||
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
|
||||
dependencies = [
|
||||
{ name = "backoff" },
|
||||
{ name = "httpx" },
|
||||
{ name = "openai" },
|
||||
{ name = "opentelemetry-api" },
|
||||
{ name = "opentelemetry-exporter-otlp-proto-http" },
|
||||
{ name = "opentelemetry-sdk" },
|
||||
{ name = "packaging" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "requests" },
|
||||
{ name = "wrapt" },
|
||||
]
|
||||
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ec/6b/7a945e8bc56cbf343b6f6171fd45870b0ea80ea38463b2db8dd5a9dc04a2/langfuse-3.14.5.tar.gz", hash = "sha256:2f543ec1540053d39b08a50ed5992caf1cd54d472a55cb8e5dcf6d4fcb7ff631", size = 235474, upload-time = "2026-02-23T10:42:47.721Z" }
|
||||
wheels = [
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/a3/a1/10f04224542d6a57073c4f339b6763836a0899c98966f1d4ffcf56d2cf61/langfuse-3.14.5-py3-none-any.whl", hash = "sha256:5054b1c705ec69bce2d7077ce7419727ac629159428da013790979ca9cae77d5", size = 421240, upload-time = "2026-02-23T10:42:46.085Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "langgraph"
|
||||
version = "1.0.6"
|
||||
@ -3132,35 +3081,6 @@ wheels = [
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/e7/c3/26b8a0908a9db249de3b4169692e1c7c19048a9bc41a4d3209cee7dbb758/psycopg_pool-3.3.0-py3-none-any.whl", hash = "sha256:2e44329155c410b5e8666372db44276a8b1ebd8c90f1c3026ebba40d4bc81063", size = 39995, upload-time = "2025-12-01T11:34:29.761Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyarrow"
|
||||
version = "23.0.1"
|
||||
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
|
||||
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/88/22/134986a4cc224d593c1afde5494d18ff629393d74cc2eddb176669f234a4/pyarrow-23.0.1.tar.gz", hash = "sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019", size = 1167336, upload-time = "2026-02-16T10:14:12.39Z" }
|
||||
wheels = [
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/9a/4b/4166bb5abbfe6f750fc60ad337c43ecf61340fa52ab386da6e8dbf9e63c4/pyarrow-23.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f4b0dbfa124c0bb161f8b5ebb40f1a680b70279aa0c9901d44a2b5a20806039f", size = 34214575, upload-time = "2026-02-16T10:09:56.225Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/e1/da/3f941e3734ac8088ea588b53e860baeddac8323ea40ce22e3d0baa865cc9/pyarrow-23.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:7707d2b6673f7de054e2e83d59f9e805939038eebe1763fe811ee8fa5c0cd1a7", size = 35832540, upload-time = "2026-02-16T10:10:03.428Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/88/7c/3d841c366620e906d54430817531b877ba646310296df42ef697308c2705/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:86ff03fb9f1a320266e0de855dee4b17da6794c595d207f89bba40d16b5c78b9", size = 44470940, upload-time = "2026-02-16T10:10:10.704Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/2c/a5/da83046273d990f256cb79796a190bbf7ec999269705ddc609403f8c6b06/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:813d99f31275919c383aab17f0f455a04f5a429c261cc411b1e9a8f5e4aaaa05", size = 47586063, upload-time = "2026-02-16T10:10:17.95Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/5b/3c/b7d2ebcff47a514f47f9da1e74b7949138c58cfeb108cdd4ee62f43f0cf3/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bf5842f960cddd2ef757d486041d57c96483efc295a8c4a0e20e704cbbf39c67", size = 48173045, upload-time = "2026-02-16T10:10:25.363Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/43/b2/b40961262213beaba6acfc88698eb773dfce32ecdf34d19291db94c2bd73/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564baf97c858ecc03ec01a41062e8f4698abc3e6e2acd79c01c2e97880a19730", size = 50621741, upload-time = "2026-02-16T10:10:33.477Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/f6/70/1fdda42d65b28b078e93d75d371b2185a61da89dda4def8ba6ba41ebdeb4/pyarrow-23.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:07deae7783782ac7250989a7b2ecde9b3c343a643f82e8a4df03d93b633006f0", size = 27620678, upload-time = "2026-02-16T10:10:39.31Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/47/10/2cbe4c6f0fb83d2de37249567373d64327a5e4d8db72f486db42875b08f6/pyarrow-23.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6b8fda694640b00e8af3c824f99f789e836720aa8c9379fb435d4c4953a756b8", size = 34210066, upload-time = "2026-02-16T10:10:45.487Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/cb/4f/679fa7e84dadbaca7a65f7cdba8d6c83febbd93ca12fa4adf40ba3b6362b/pyarrow-23.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:8ff51b1addc469b9444b7c6f3548e19dc931b172ab234e995a60aea9f6e6025f", size = 35825526, upload-time = "2026-02-16T10:10:52.266Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/f9/63/d2747d930882c9d661e9398eefc54f15696547b8983aaaf11d4a2e8b5426/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:71c5be5cbf1e1cb6169d2a0980850bccb558ddc9b747b6206435313c47c37677", size = 44473279, upload-time = "2026-02-16T10:11:01.557Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/b3/93/10a48b5e238de6d562a411af6467e71e7aedbc9b87f8d3a35f1560ae30fb/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9b6f4f17b43bc39d56fec96e53fe89d94bac3eb134137964371b45352d40d0c2", size = 47585798, upload-time = "2026-02-16T10:11:09.401Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/5c/20/476943001c54ef078dbf9542280e22741219a184a0632862bca4feccd666/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fc13fc6c403d1337acab46a2c4346ca6c9dec5780c3c697cf8abfd5e19b6b37", size = 48179446, upload-time = "2026-02-16T10:11:17.781Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/4b/b6/5dd0c47b335fcd8edba9bfab78ad961bd0fd55ebe53468cc393f45e0be60/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c16ed4f53247fa3ffb12a14d236de4213a4415d127fe9cebed33d51671113e2", size = 50623972, upload-time = "2026-02-16T10:11:26.185Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/d5/09/a532297c9591a727d67760e2e756b83905dd89adb365a7f6e9c72578bcc1/pyarrow-23.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:cecfb12ef629cf6be0b1887f9f86463b0dd3dc3195ae6224e74006be4736035a", size = 27540749, upload-time = "2026-02-16T10:12:23.297Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/a5/8e/38749c4b1303e6ae76b3c80618f84861ae0c55dd3c2273842ea6f8258233/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:29f7f7419a0e30264ea261fdc0e5fe63ce5a6095003db2945d7cd78df391a7e1", size = 34471544, upload-time = "2026-02-16T10:11:32.535Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/a3/73/f237b2bc8c669212f842bcfd842b04fc8d936bfc9d471630569132dc920d/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:33d648dc25b51fd8055c19e4261e813dfc4d2427f068bcecc8b53d01b81b0500", size = 35949911, upload-time = "2026-02-16T10:11:39.813Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/0c/86/b912195eee0903b5611bf596833def7d146ab2d301afeb4b722c57ffc966/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd395abf8f91c673dd3589cadc8cc1ee4e8674fa61b2e923c8dd215d9c7d1f41", size = 44520337, upload-time = "2026-02-16T10:11:47.764Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/69/c2/f2a717fb824f62d0be952ea724b4f6f9372a17eed6f704b5c9526f12f2f1/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:00be9576d970c31defb5c32eb72ef585bf600ef6d0a82d5eccaae96639cf9d07", size = 47548944, upload-time = "2026-02-16T10:11:56.607Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/84/a7/90007d476b9f0dc308e3bc57b832d004f848fd6c0da601375d20d92d1519/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c2139549494445609f35a5cda4eb94e2c9e4d704ce60a095b342f82460c73a83", size = 48236269, upload-time = "2026-02-16T10:12:04.47Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/b0/3f/b16fab3e77709856eb6ac328ce35f57a6d4a18462c7ca5186ef31b45e0e0/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7044b442f184d84e2351e5084600f0d7343d6117aabcbc1ac78eb1ae11eb4125", size = 50604794, upload-time = "2026-02-16T10:12:11.797Z" },
|
||||
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/e9/a1/22df0620a9fac31d68397a75465c344e83c3dfe521f7612aea33e27ab6c0/pyarrow-23.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a35581e856a2fafa12f3f54fce4331862b1cfb0bef5758347a858a4aa9d6bae8", size = 27660642, upload-time = "2026-02-16T10:12:17.746Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyasn1"
|
||||
version = "0.6.1"
|
||||
@ -5161,7 +5081,6 @@ dependencies = [
|
||||
{ name = "langchain-openai" },
|
||||
{ name = "langchain-tavily" },
|
||||
{ name = "langchain-text-splitters" },
|
||||
{ name = "langfuse" },
|
||||
{ name = "langgraph" },
|
||||
{ name = "langgraph-checkpoint-postgres" },
|
||||
{ name = "langgraph-checkpoint-sqlite" },
|
||||
@ -5212,10 +5131,6 @@ dependencies = [
|
||||
dev = [
|
||||
{ name = "ruff" },
|
||||
]
|
||||
eval = [
|
||||
{ name = "datasets" },
|
||||
{ name = "huggingface-hub" },
|
||||
]
|
||||
test = [
|
||||
{ name = "pytest" },
|
||||
{ name = "pytest-asyncio" },
|
||||
@ -5247,7 +5162,6 @@ requires-dist = [
|
||||
{ name = "langchain-openai", specifier = ">=1.0.2" },
|
||||
{ name = "langchain-tavily", specifier = ">=0.2.13" },
|
||||
{ name = "langchain-text-splitters", specifier = ">=1.0" },
|
||||
{ name = "langfuse", specifier = ">=2.0.0" },
|
||||
{ name = "langgraph", specifier = ">=1.0.1" },
|
||||
{ name = "langgraph-checkpoint-postgres", specifier = ">=2.0.0" },
|
||||
{ name = "langgraph-checkpoint-sqlite", specifier = ">=3.0" },
|
||||
@ -5294,10 +5208,6 @@ requires-dist = [
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
dev = [{ name = "ruff", specifier = ">=0.12.1" }]
|
||||
eval = [
|
||||
{ name = "datasets", specifier = ">=3.0.0" },
|
||||
{ name = "huggingface-hub", specifier = ">=0.25.0" },
|
||||
]
|
||||
test = [
|
||||
{ name = "pytest", specifier = ">=8.0.0" },
|
||||
{ name = "pytest-asyncio", specifier = ">=0.24.0" },
|
||||
|
||||
Loading…
Reference in New Issue
Block a user