Merge branch 'codex/feature-langfuse-gaia-eval'

docker-compose.yml

@@ -13,6 +13,8 @@ services:
       - ./test:/app/test
       - ./scripts:/app/scripts
       - ./.env:/app/.env
+      - ./pyproject.toml:/app/pyproject.toml
+      - ./uv.lock:/app/uv.lock
       - ${MODEL_DIR:-./models}:/models  # 使用默认值处理未定义的环境变量
     ports:
       - "5050:5050"

pyproject.toml

@@ -66,6 +66,7 @@ dependencies = [
     "torchvision==0.23",
     "docling>=2.68.0",
     "loguru>=0.7.3",
+    "langfuse>=2.0.0",
 ]
 [tool.ruff]
 line-length = 120  # 代码最大行宽
@@ -116,3 +117,7 @@ test = [
     "pytest-httpx>=0.32.0",
     "pytest-cov>=6.0.0",
 ]
+eval = [
+    "datasets>=3.0.0",
+    "huggingface-hub>=0.25.0",
+]

src/agents/common/base.py

@@ -13,6 +13,7 @@ from langgraph.graph.state import CompiledStateGraph
 from src import config as sys_config
 from src.agents.common.context import BaseContext
 from src.utils import logger
+from src.utils.langfuse_integration import get_langfuse_callback
 
 
 class BaseAgent:
@@ -68,7 +69,13 @@ class BaseAgent:
         if isinstance(agent_config, dict):
             context.update(agent_config)
         context.update(input_context or {})
-        for event in graph.astream({"messages": messages}, stream_mode="values", context=context):
+
+        stream_config = {}
+        langfuse_handler = get_langfuse_callback()
+        if langfuse_handler:
+            stream_config["callbacks"] = [langfuse_handler]
+
+        for event in graph.astream({"messages": messages}, stream_mode="values", context=context, config=stream_config):
             yield event["messages"]
 
     async def stream_messages(self, messages: list[str], input_context=None, **kwargs):
@@ -85,6 +92,9 @@ class BaseAgent:
             "configurable": {"thread_id": context.thread_id, "user_id": context.user_id},
             "recursion_limit": 300,
         }
+        langfuse_handler = get_langfuse_callback()
+        if langfuse_handler:
+            input_config["callbacks"] = [langfuse_handler]
 
         async for msg, metadata in graph.astream(
             {"messages": messages},
@@ -108,6 +118,9 @@ class BaseAgent:
             "configurable": {"thread_id": context.thread_id, "user_id": context.user_id},
             "recursion_limit": 100,
         }
+        langfuse_handler = get_langfuse_callback()
+        if langfuse_handler:
+            input_config["callbacks"] = [langfuse_handler]
 
         msg = await graph.ainvoke(
             {"messages": messages},

src/models/chat.py

@@ -1,11 +1,16 @@
 import os
 import traceback
 
-from openai import AsyncOpenAI
 from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_exponential
 
 from src import config
 from src.utils import logger
+from src.utils.langfuse_integration import is_langfuse_enabled
+
+if is_langfuse_enabled():
+    from langfuse.openai import AsyncOpenAI
+else:
+    from openai import AsyncOpenAI
 
 
 def split_model_spec(model_spec, sep="/"):

src/utils/langfuse_integration.py

@@ -0,0 +1,39 @@
+"""
+Langfuse 集成模块
+
+提供 Langfuse 的初始化和回调处理器获取功能。
+仅当环境变量 LANGFUSE_SECRET_KEY 和 LANGFUSE_PUBLIC_KEY 存在时启用。
+"""
+
+import os
+from functools import lru_cache
+
+from src.utils.logging_config import logger
+
+
+def is_langfuse_enabled() -> bool:
+    """检查 Langfuse 是否已配置"""
+    return bool(
+        os.getenv("LANGFUSE_SECRET_KEY")
+        and os.getenv("LANGFUSE_PUBLIC_KEY")
+    )
+
+
+@lru_cache(maxsize=1)
+def get_langfuse_callback():
+    """获取 Langfuse LangChain CallbackHandler（单例）
+
+    Returns:
+        CallbackHandler 实例，若 Langfuse 未配置则返回 None
+    """
+    if not is_langfuse_enabled():
+        return None
+    try:
+        from langfuse.langchain import CallbackHandler
+
+        handler = CallbackHandler()
+        logger.info("Langfuse CallbackHandler 初始化成功")
+        return handler
+    except Exception as e:
+        logger.warning(f"Langfuse CallbackHandler 初始化失败: {e}")
+        return None

test/gaia_eval/__init__.py

@@ -0,0 +1,21 @@
+"""GAIA 离线评估框架
+
+使用 GAIA benchmark 评估 Yuxi-Know 的 Agent 系统。
+
+用法:
+    python -m test.gaia_eval --level 1 --max-samples 5
+"""
+
+from .config import EvalConfig
+from .dataset_loader import GaiaDatasetLoader, GaiaTask
+from .runner import EvalResult, GaiaEvalRunner
+from .scorer import GaiaScorer
+
+__all__ = [
+    "EvalConfig",
+    "GaiaDatasetLoader",
+    "GaiaTask",
+    "GaiaEvalRunner",
+    "EvalResult",
+    "GaiaScorer",
+]

test/gaia_eval/__main__.py

@@ -0,0 +1,109 @@
+"""GAIA 离线评估 CLI 入口
+
+用法:
+    python -m test.gaia_eval                                   # 评估全部级别
+    python -m test.gaia_eval --level 1                         # 只评估 Level 1
+    python -m test.gaia_eval --level 1 --max-samples 5         # 限制 5 条（调试）
+    python -m test.gaia_eval --agent-config-id 3               # 使用数据库预设配置
+    python -m test.gaia_eval --model deepseek-chat --level 1   # 指定模型
+    python -m test.gaia_eval --output-dir ./my_results         # 自定义输出目录
+"""
+
+import argparse
+import asyncio
+import sys
+
+from .config import EvalConfig
+from .dataset_loader import GaiaDatasetLoader
+from .reporter import GaiaReporter
+from .runner import GaiaEvalRunner
+
+
+def parse_args() -> EvalConfig:
+    """解析命令行参数"""
+    parser = argparse.ArgumentParser(
+        description="GAIA 离线评估 - 评估 Yuxi-Know Agent 系统",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    parser.add_argument("--agent-id", default="ChatbotAgent", help="Agent ID（默认: ChatbotAgent）")
+    parser.add_argument("--agent-config-id", type=int, default=None, help="Agent 预设配置 ID，从数据库加载")
+    parser.add_argument("--model", default=None, help="模型名称（当未指定 agent-config-id 时使用）")
+    parser.add_argument("--level", default="all", choices=["1", "2", "3", "all"], help="评估级别（默认: all）")
+    parser.add_argument("--split", default="validation", choices=["validation", "test"], help="数据集划分（默认: validation）")
+    parser.add_argument("--max-samples", type=int, default=None, help="最大评估样本数（调试用）")
+    parser.add_argument("--timeout", type=int, default=300, help="单题超时，秒（默认: 300）")
+    parser.add_argument("--concurrency", type=int, default=1, help="并发评估数（默认: 1）")
+    parser.add_argument("--output-dir", default="eval_results", help="结果输出目录（默认: eval_results）")
+
+    args = parser.parse_args()
+
+    from pathlib import Path
+
+    return EvalConfig(
+        agent_id=args.agent_id,
+        agent_config_id=args.agent_config_id,
+        model=args.model,
+        level=args.level,
+        split=args.split,
+        max_samples=args.max_samples,
+        timeout=args.timeout,
+        concurrency=args.concurrency,
+        output_dir=Path(args.output_dir),
+    )
+
+
+async def main():
+    """主流程"""
+    from rich.console import Console
+
+    console = Console()
+    config = parse_args()
+
+    # 1. 显示配置信息
+    console.print("\n[bold cyan]═══ GAIA 离线评估 ═══[/bold cyan]")
+    console.print(f"  Agent:       {config.agent_id}")
+    if config.agent_config_id:
+        console.print(f"  配置ID:      {config.agent_config_id}")
+    if config.model:
+        console.print(f"  模型:        {config.model}")
+    console.print(f"  级别:        {config.level}")
+    console.print(f"  数据集:      {config.split}")
+    if config.max_samples:
+        console.print(f"  样本限制:    {config.max_samples}")
+    console.print(f"  超时:        {config.timeout}s")
+    console.print(f"  并发:        {config.concurrency}")
+    console.print(f"  输出目录:    {config.output_dir}")
+    console.print()
+
+    # 2. 加载数据集
+    console.print("[bold]正在加载 GAIA 数据集...[/bold]")
+    try:
+        loader = GaiaDatasetLoader(config)
+        tasks = loader.load_tasks()
+    except Exception as e:
+        console.print(f"[red]数据集加载失败: {e}[/red]")
+        console.print("[dim]请确保已设置 HF_TOKEN 并同意了数据集使用条款[/dim]")
+        sys.exit(1)
+
+    console.print(f"  已加载 {len(tasks)} 条评估任务\n")
+
+    if not tasks:
+        console.print("[yellow]没有可评估的任务[/yellow]")
+        sys.exit(0)
+
+    # 3. 执行评估
+    console.print("[bold]开始评估...[/bold]\n")
+    runner = GaiaEvalRunner(config)
+    results = await runner.run(tasks)
+
+    # 4. 生成报告
+    reporter = GaiaReporter(config, results)
+    reporter.print_report()
+
+    report_path = reporter.save_json_report()
+    console.print(f"[green]详细报告已保存: {report_path}[/green]\n")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

test/gaia_eval/config.py

@@ -0,0 +1,57 @@
+"""GAIA 评估配置管理模块"""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass
+class EvalConfig:
+    """GAIA 评估配置
+
+    配置优先级：CLI 参数 > 环境变量 > 默认值
+    当指定 agent_config_id 时，会从数据库加载预设配置（model/tools/knowledges/system_prompt 等）
+    """
+
+    # Agent 相关
+    agent_id: str = "ChatbotAgent"
+    agent_config_id: int | None = None  # 从数据库加载预设配置
+
+    # 备选配置（当未指定 agent_config_id 时使用）
+    model: str | None = None
+    system_prompt: str | None = None
+    tools: list[dict] | None = None
+    knowledges: list[str] | None = None
+
+    # 数据集相关
+    level: str = "all"  # "1", "2", "3", "all"
+    split: str = "validation"  # "validation" or "test"
+    max_samples: int | None = None  # 限制评估样本数（调试用）
+
+    # 运行控制
+    timeout: int = 300  # 单题超时，秒
+    concurrency: int = 1  # 并发评估数
+
+    # 输出相关
+    output_dir: Path = field(default_factory=lambda: Path("eval_results"))
+
+    def get_dataset_configs(self) -> list[str]:
+        """根据 level 获取 HuggingFace dataset config 名称列表"""
+        if self.level == "all":
+            return ["2023_level1", "2023_level2", "2023_level3"]
+        return [f"2023_level{self.level}"]
+
+    def build_agent_config(self) -> dict:
+        """构建不使用 agent_config_id 时的 agent_config 字典
+
+        当未指定 agent_config_id 时，从 CLI 参数组装配置。
+        """
+        config = {}
+        if self.model:
+            config["model"] = self.model
+        if self.system_prompt:
+            config["system_prompt"] = self.system_prompt
+        if self.tools is not None:
+            config["tools"] = self.tools
+        if self.knowledges is not None:
+            config["knowledges"] = self.knowledges
+        return config

test/gaia_eval/dataset_loader.py

@@ -0,0 +1,164 @@
+"""GAIA 数据集加载器
+
+支持两种模式：
+  1. 流式模式（默认）：直接 streaming 拉取 metadata，无需完整下载，适合快速测试
+  2. 完整模式：snapshot_download 后加载，附件文件可本地读取
+
+需要设置 HF_TOKEN 环境变量，并在 HuggingFace 上同意数据集使用条款。
+"""
+
+import os
+from dataclasses import dataclass
+
+from .config import EvalConfig
+
+
+@dataclass
+class GaiaTask:
+    """单条 GAIA 评估任务"""
+
+    task_id: str
+    question: str
+    level: int
+    final_answer: str
+    file_name: str | None = None
+    file_path: str | None = None  # 流式模式下为 None（无本地文件）
+    annotator_metadata: dict | None = None
+
+
+def _get_token() -> str | None:
+    """获取 HF_TOKEN，支持从环境变量或 .env 文件读取"""
+    token = os.environ.get("HF_TOKEN")
+    if not token:
+        try:
+            from dotenv import load_dotenv
+            load_dotenv()
+            token = os.environ.get("HF_TOKEN")
+        except ImportError:
+            pass
+    return token
+
+
+class GaiaDatasetLoader:
+    """GAIA 数据集加载器"""
+
+    REPO_ID = "gaia-benchmark/GAIA"
+
+    def __init__(self, config: EvalConfig):
+        self.config = config
+        self._data_dir: str | None = None  # 完整下载后的本地路径
+
+    def load_tasks(self) -> list[GaiaTask]:
+        """加载并返回 GaiaTask 列表
+
+        当 max_samples 较小时，自动使用流式加载避免完整下载；
+        否则使用 snapshot_download 完整下载（支持附件文件读取）。
+        """
+        # 小样本时用流式加载，避免下载完整数据集
+        small_sample = self.config.max_samples and self.config.max_samples <= 50
+        if small_sample:
+            return self._load_streaming()
+        else:
+            return self._load_full()
+
+    def _load_streaming(self) -> list[GaiaTask]:
+        """流式加载：直接从 HuggingFace 流式拉取，附件文件按需单独下载"""
+        from datasets import load_dataset
+        from huggingface_hub import hf_hub_download
+
+        token = _get_token()
+        if not token:
+            raise RuntimeError(
+                "未设置 HF_TOKEN 环境变量。请在 .env 中添加 HF_TOKEN=hf_xxx"
+            )
+
+        dataset_configs = self.config.get_dataset_configs()
+        tasks: list[GaiaTask] = []
+
+        for ds_config in dataset_configs:
+            dataset = load_dataset(
+                self.REPO_ID,
+                ds_config,
+                split=self.config.split,
+                streaming=True,
+                token=token,
+            )
+
+            for example in dataset:
+                file_name = example.get("file_name") or None
+                file_path = None
+
+                # 按需下载单个附件文件（只下载当前任务的文件，不下载整个数据集）
+                if file_name:
+                    repo_file_path = f"2023/{self.config.split}/{file_name}"
+                    try:
+                        file_path = hf_hub_download(
+                            repo_id=self.REPO_ID,
+                            repo_type="dataset",
+                            filename=repo_file_path,
+                            token=token,
+                        )
+                    except Exception as e:
+                        import warnings
+                        warnings.warn(f"附件下载失败 {file_name}: {e}")
+
+                task = GaiaTask(
+                    task_id=example["task_id"],
+                    question=example["Question"],
+                    level=int(example["Level"]),
+                    final_answer=example.get("Final answer", ""),
+                    file_name=file_name,
+                    file_path=file_path,
+                    annotator_metadata=example.get("Annotator Metadata"),
+                )
+                tasks.append(task)
+
+                if self.config.max_samples and len(tasks) >= self.config.max_samples:
+                    return tasks
+
+        return tasks
+
+    def _load_full(self) -> list[GaiaTask]:
+        """完整下载：先 snapshot_download，再加载（附件文件可本地读取）"""
+        from datasets import load_dataset
+        from huggingface_hub import snapshot_download
+
+        token = _get_token()
+        if not token:
+            raise RuntimeError(
+                "未设置 HF_TOKEN 环境变量。请在 .env 中添加 HF_TOKEN=hf_xxx"
+            )
+
+        if not self._data_dir:
+            self._data_dir = snapshot_download(
+                repo_id=self.REPO_ID,
+                repo_type="dataset",
+                token=token,
+            )
+
+        dataset_configs = self.config.get_dataset_configs()
+        tasks: list[GaiaTask] = []
+
+        for ds_config in dataset_configs:
+            dataset = load_dataset(self._data_dir, ds_config, split=self.config.split)
+
+            for example in dataset:
+                file_path = example.get("file_path")
+                if file_path:
+                    file_path = os.path.join(self._data_dir, file_path)
+
+                task = GaiaTask(
+                    task_id=example["task_id"],
+                    question=example["Question"],
+                    level=int(example["Level"]),
+                    final_answer=example.get("Final answer", ""),
+                    file_name=example.get("file_name") or None,
+                    file_path=file_path,
+                    annotator_metadata=example.get("Annotator Metadata"),
+                )
+                tasks.append(task)
+
+        if self.config.max_samples and len(tasks) > self.config.max_samples:
+            tasks = tasks[: self.config.max_samples]
+
+        return tasks

test/gaia_eval/file_handler.py

@@ -0,0 +1,191 @@
+"""GAIA 评估文件处理模块
+
+将 GAIA 数据集中的附件文件转换为 Agent 可识别的 attachments + state files 格式。
+"""
+
+from __future__ import annotations
+
+import base64
+import mimetypes
+import os
+from datetime import datetime
+
+from src.utils import logger
+
+
+# 支持直接读取内容的文本类文件扩展名
+TEXT_EXTENSIONS = {
+    ".txt", ".md", ".csv", ".tsv", ".json", ".jsonl",
+    ".xml", ".html", ".htm", ".yaml", ".yml", ".log",
+    ".py", ".js", ".ts", ".java", ".c", ".cpp", ".h",
+    ".rb", ".go", ".rs", ".sh", ".bat", ".sql",
+    ".ini", ".cfg", ".conf", ".toml",
+}
+
+# 支持用 PDF 解析的扩展名
+PDF_EXTENSIONS = {".pdf"}
+
+# 支持用 pandas 等读取的表格扩展名
+SPREADSHEET_EXTENSIONS = {".xlsx", ".xls"}
+
+# 图片类扩展名（用于多模态消息）
+IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp"}
+
+
+def _read_text_file(file_path: str) -> str | None:
+    """读取纯文本文件"""
+    encodings = ["utf-8", "latin-1", "gbk"]
+    for enc in encodings:
+        try:
+            with open(file_path, encoding=enc) as f:
+                return f.read()
+        except (UnicodeDecodeError, OSError):
+            continue
+    return None
+
+
+def _read_pdf_file(file_path: str) -> str | None:
+    """解析 PDF 文件为文本"""
+    try:
+        import fitz  # PyMuPDF
+
+        doc = fitz.open(file_path)
+        pages = []
+        for page in doc:
+            pages.append(page.get_text())
+        doc.close()
+        return "\n\n".join(pages)
+    except ImportError:
+        logger.warning("PyMuPDF (fitz) 未安装，无法解析 PDF。请安装: pip install PyMuPDF")
+        return None
+    except Exception as e:
+        logger.warning(f"PDF 解析失败 {file_path}: {e}")
+        return None
+
+
+def _read_spreadsheet_file(file_path: str) -> str | None:
+    """读取 Excel 文件并转为 CSV 文本"""
+    try:
+        import pandas as pd
+
+        df = pd.read_excel(file_path)
+        return df.to_csv(index=False)
+    except ImportError:
+        logger.warning("pandas/openpyxl 未安装，无法解析 Excel。请安装: pip install pandas openpyxl")
+        return None
+    except Exception as e:
+        logger.warning(f"Excel 解析失败 {file_path}: {e}")
+        return None
+
+
+def _read_image_as_base64(file_path: str) -> str | None:
+    """读取图片为 base64"""
+    try:
+        with open(file_path, "rb") as f:
+            data = f.read()
+        return base64.b64encode(data).decode("utf-8")
+    except Exception as e:
+        logger.warning(f"图片读取失败 {file_path}: {e}")
+        return None
+
+
+def read_file_content(file_path: str) -> str | None:
+    """根据文件类型读取内容，返回文本"""
+    if not file_path or not os.path.exists(file_path):
+        logger.warning(f"文件不存在: {file_path}")
+        return None
+
+    _, ext = os.path.splitext(file_path)
+    ext = ext.lower()
+
+    if ext in TEXT_EXTENSIONS:
+        return _read_text_file(file_path)
+    elif ext in PDF_EXTENSIONS:
+        return _read_pdf_file(file_path)
+    elif ext in SPREADSHEET_EXTENSIONS:
+        return _read_spreadsheet_file(file_path)
+    else:
+        # 尝试作为文本读取
+        content = _read_text_file(file_path)
+        if content:
+            return content
+        logger.warning(f"不支持的文件类型: {ext} ({file_path})")
+        return None
+
+
+def build_attachments_and_files(
+    file_path: str,
+    file_name: str | None = None,
+) -> tuple[list[dict], dict]:
+    """将文件转换为 Agent 的 attachments 和 state files 格式
+
+    返回:
+        (attachments, files) 元组：
+        - attachments: 附件元数据列表，用于 AttachmentMiddleware
+        - files: StateBackend 格式的文件字典，用于 read_file 工具
+    """
+    if not file_path or not os.path.exists(file_path):
+        return [], {}
+
+    if not file_name:
+        file_name = os.path.basename(file_path)
+
+    _, ext = os.path.splitext(file_path)
+    ext = ext.lower()
+
+    # 图片使用多模态消息，不走 attachment 机制
+    if ext in IMAGE_EXTENSIONS:
+        return [], {}
+
+    content = read_file_content(file_path)
+    if not content:
+        return [], {}
+
+    now = datetime.utcnow().isoformat() + "+00:00"
+    virtual_path = f"/attachments/{file_name}"
+
+    # 构建 attachment 元数据
+    attachment = {
+        "file_name": file_name,
+        "file_path": virtual_path,
+        "status": "parsed",
+        "markdown": content,
+        "uploaded_at": now,
+        "truncated": False,
+    }
+
+    # 构建 state files（与 _build_state_files 格式一致）
+    content_lines = content.split("\n")
+    files = {
+        virtual_path: {
+            "content": content_lines,
+            "created_at": now,
+            "modified_at": now,
+        }
+    }
+
+    return [attachment], files
+
+
+def build_image_message_content(file_path: str, question: str) -> list[dict] | None:
+    """为图片文件构建多模态消息内容
+
+    返回 HumanMessage 的 content（多模态格式），或 None 如果不是图片。
+    """
+    if not file_path or not os.path.exists(file_path):
+        return None
+
+    _, ext = os.path.splitext(file_path)
+    if ext.lower() not in IMAGE_EXTENSIONS:
+        return None
+
+    base64_data = _read_image_as_base64(file_path)
+    if not base64_data:
+        return None
+
+    mime_type = mimetypes.guess_type(file_path)[0] or "image/jpeg"
+
+    return [
+        {"type": "text", "text": question},
+        {"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{base64_data}"}},
+    ]

test/gaia_eval/reporter.py

@@ -0,0 +1,162 @@
+"""GAIA 评估报告生成模块
+
+输出终端彩色表格和 JSON 详细报告。
+"""
+
+import json
+from collections import defaultdict
+from datetime import datetime
+from pathlib import Path
+
+from .config import EvalConfig
+from .runner import EvalResult
+
+
+class GaiaReporter:
+    """评估报告生成器"""
+
+    def __init__(self, config: EvalConfig, results: list[EvalResult]):
+        self.config = config
+        self.results = results
+
+    def _compute_stats(self) -> dict:
+        """计算统计数据"""
+        total = len(self.results)
+        correct = sum(1 for r in self.results if r.is_correct)
+        errors = sum(1 for r in self.results if r.error is not None)
+
+        # 按 Level 分组统计
+        by_level: dict[int, dict] = defaultdict(lambda: {"total": 0, "correct": 0, "errors": 0})
+        for r in self.results:
+            by_level[r.level]["total"] += 1
+            if r.is_correct:
+                by_level[r.level]["correct"] += 1
+            if r.error:
+                by_level[r.level]["errors"] += 1
+
+        # 耗时统计
+        durations = [r.duration_seconds for r in self.results if not r.error]
+        avg_duration = sum(durations) / len(durations) if durations else 0
+        max_duration = max(durations) if durations else 0
+        min_duration = min(durations) if durations else 0
+
+        return {
+            "total": total,
+            "correct": correct,
+            "accuracy": round(correct / total * 100, 2) if total > 0 else 0,
+            "errors": errors,
+            "by_level": {
+                level: {
+                    "total": stats["total"],
+                    "correct": stats["correct"],
+                    "accuracy": round(stats["correct"] / stats["total"] * 100, 2) if stats["total"] > 0 else 0,
+                    "errors": stats["errors"],
+                }
+                for level, stats in sorted(by_level.items())
+            },
+            "duration": {
+                "avg": round(avg_duration, 2),
+                "max": round(max_duration, 2),
+                "min": round(min_duration, 2),
+                "total": round(sum(durations), 2),
+            },
+        }
+
+    def print_report(self):
+        """在终端输出彩色报告"""
+        from rich.console import Console
+        from rich.table import Table
+
+        console = Console()
+        stats = self._compute_stats()
+
+        # 标题
+        console.print("\n[bold cyan]═══ GAIA 评估报告 ═══[/bold cyan]\n")
+
+        # 总体统计
+        accuracy_color = "green" if stats["accuracy"] >= 50 else ("yellow" if stats["accuracy"] >= 25 else "red")
+        console.print(f"  Agent:     [bold]{self.config.agent_id}[/bold]")
+        if self.config.agent_config_id:
+            console.print(f"  配置ID:    [bold]{self.config.agent_config_id}[/bold]")
+        console.print(f"  总题数:    {stats['total']}")
+        console.print(f"  正确数:    {stats['correct']}")
+        console.print(f"  错误数:    {stats['errors']}")
+        console.print(f"  准确率:    [{accuracy_color}]{stats['accuracy']}%[/{accuracy_color}]")
+        console.print()
+
+        # 按 Level 统计表
+        level_table = Table(title="按 Level 统计")
+        level_table.add_column("Level", style="bold")
+        level_table.add_column("总数", justify="right")
+        level_table.add_column("正确", justify="right")
+        level_table.add_column("准确率", justify="right")
+        level_table.add_column("错误", justify="right")
+
+        for level, level_stats in stats["by_level"].items():
+            acc_color = "green" if level_stats["accuracy"] >= 50 else (
+                "yellow" if level_stats["accuracy"] >= 25 else "red"
+            )
+            level_table.add_row(
+                f"Level {level}",
+                str(level_stats["total"]),
+                str(level_stats["correct"]),
+                f"[{acc_color}]{level_stats['accuracy']}%[/{acc_color}]",
+                str(level_stats["errors"]),
+            )
+
+        console.print(level_table)
+        console.print()
+
+        # 耗时统计
+        console.print("[bold]耗时统计:[/bold]")
+        console.print(f"  平均: {stats['duration']['avg']}s")
+        console.print(f"  最大: {stats['duration']['max']}s")
+        console.print(f"  最小: {stats['duration']['min']}s")
+        console.print(f"  总计: {stats['duration']['total']}s")
+        console.print()
+
+        # 错误样本
+        error_results = [r for r in self.results if r.error]
+        if error_results:
+            console.print(f"[bold red]错误样本 ({len(error_results)} 条):[/bold red]")
+            for r in error_results[:5]:
+                console.print(f"  • [{r.task_id[:8]}] L{r.level}: {r.error}")
+            if len(error_results) > 5:
+                console.print(f"  ... 还有 {len(error_results) - 5} 条错误")
+            console.print()
+
+    def save_json_report(self) -> Path:
+        """保存 JSON 详细报告"""
+        self.config.output_dir.mkdir(parents=True, exist_ok=True)
+
+        stats = self._compute_stats()
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        report_path = self.config.output_dir / f"gaia_eval_{timestamp}.json"
+
+        report = {
+            "metadata": {
+                "timestamp": timestamp,
+                "agent_id": self.config.agent_id,
+                "agent_config_id": self.config.agent_config_id,
+                "level": self.config.level,
+                "split": self.config.split,
+                "timeout": self.config.timeout,
+            },
+            "summary": stats,
+            "results": [
+                {
+                    "task_id": r.task_id,
+                    "level": r.level,
+                    "question": r.question,
+                    "gold_answer": r.gold_answer,
+                    "predicted_answer": r.predicted_answer,
+                    "is_correct": r.is_correct,
+                    "error": r.error,
+                    "duration_seconds": r.duration_seconds,
+                }
+                for r in self.results
+            ],
+        }
+
+        report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
+        return report_path

test/gaia_eval/runner.py

@@ -0,0 +1,272 @@
+"""GAIA 评估运行器
+
+核心评估逻辑：加载 agent、执行评估、收集结果。
+"""
+
+import asyncio
+import time
+import uuid
+from dataclasses import dataclass
+
+from tqdm import tqdm
+
+from .config import EvalConfig
+from .dataset_loader import GaiaTask
+from .scorer import GaiaScorer
+
+
+@dataclass
+class EvalResult:
+    """单条评估结果"""
+
+    task_id: str
+    level: int
+    question: str
+    gold_answer: str
+    predicted_answer: str
+    is_correct: bool
+    error: str | None = None
+    duration_seconds: float = 0.0
+
+
+class GaiaEvalRunner:
+    """GAIA 评估运行器
+
+    通过直接调用 BaseAgent.invoke_messages() 执行评估，
+    无需启动 Web 服务。
+    """
+
+    def __init__(self, config: EvalConfig):
+        self.config = config
+        self._agent = None
+        self._agent_config: dict = {}
+
+    async def _init_agent(self):
+        """初始化 Agent 实例并加载预设配置"""
+        from src.agents import agent_manager
+
+        self._agent = agent_manager.get_agent(self.config.agent_id)
+
+        # 如果指定了 agent_config_id，从数据库加载预设配置
+        if self.config.agent_config_id is not None:
+            await self._load_agent_config_from_db()
+        else:
+            # 使用 CLI 参数组装配置
+            self._agent_config = self.config.build_agent_config()
+
+    async def _load_agent_config_from_db(self):
+        """从数据库加载 agent 预设配置"""
+        from src.repositories.agent_config_repository import AgentConfigRepository
+        from src.storage.postgres.manager import pg_manager
+
+        # 评估脚本独立运行时，pg_manager 和 mcp_service 未经过 FastAPI 启动事件初始化，需手动初始化
+        if not pg_manager._initialized:
+            pg_manager.initialize()
+        
+        from src.services.mcp_service import init_mcp_servers
+        await init_mcp_servers()
+
+        async with pg_manager.get_async_session_context() as db:
+            repo = AgentConfigRepository(db)
+            config_item = await repo.get_by_id(self.config.agent_config_id)
+
+            if config_item is None:
+                raise ValueError(f"Agent 配置 ID {self.config.agent_config_id} 不存在")
+
+            # 与线上逻辑保持一致：从 config_json.context 中提取配置
+            self._agent_config = (config_item.config_json or {}).get("context", {})
+
+    def _extract_answer(self, result: dict) -> str:
+        """从 agent 返回结果中提取最终答案
+
+        Agent 返回格式为 { "messages": [...] }，取最后一条 AI 消息的内容。
+        """
+        messages = result.get("messages", [])
+        if not messages:
+            return ""
+
+        # 从后往前找第一条 AI 消息
+        for msg in reversed(messages):
+            msg_type = getattr(msg, "type", None)
+            if msg_type == "ai":
+                content = getattr(msg, "content", "")
+                if isinstance(content, str) and content.strip():
+                    return self._parse_final_answer(content)
+
+        return ""
+
+    @staticmethod
+    def _parse_final_answer(content: str) -> str:
+        """尝试从 AI 回复中解析出最终答案
+
+        如果回复中包含 "FINAL ANSWER:" 标记，则提取标记后面的内容。
+        否则返回完整回复。
+        """
+        # 常见答案标记格式
+        markers = [
+            "FINAL ANSWER:",
+            "Final Answer:",
+            "final answer:",
+            "最终答案：",
+            "最终答案:",
+            "答案：",
+            "答案:",
+        ]
+        for marker in markers:
+            if marker in content:
+                answer = content.split(marker, 1)[1].strip()
+                # 取第一行（去掉后续的解释）
+                first_line = answer.split("\n")[0].strip()
+                return first_line
+
+        # 如果只有一行，直接返回
+        lines = [ln.strip() for ln in content.strip().split("\n") if ln.strip()]
+        if len(lines) == 1:
+            return lines[0]
+
+        # 多行回复，返回最后一行（通常是最终答案）
+        return lines[-1] if lines else content.strip()
+
+    async def _run_single_task(self, task: GaiaTask) -> EvalResult:
+        """执行单条评估任务"""
+        start_time = time.time()
+        predicted_answer = ""
+        error = None
+
+        try:
+            from langchain.messages import HumanMessage
+
+            from .file_handler import build_attachments_and_files, build_image_message_content
+
+            # 构建系统提示词
+            gaia_system_suffix = (
+                "\n\nIMPORTANT: When you have found the answer, "
+                "state it clearly on a single line prefixed with 'FINAL ANSWER: '. "
+                "Keep the answer as concise as possible - just the answer itself, "
+                "no extra explanation."
+            )
+
+            input_context = {
+                "user_id": "gaia_eval",
+                "thread_id": str(uuid.uuid4()),
+                "agent_config": {
+                    **self._agent_config,
+                    "system_prompt": (
+                        self._agent_config.get("system_prompt", "You are a helpful assistant.")
+                        + gaia_system_suffix
+                    ),
+                },
+            }
+
+            # 处理附件文件
+            attachments: list[dict] = []
+            files: dict = {}
+            image_content: list[dict] | None = None
+
+            if task.file_path:
+                # 尝试构建图片多模态消息
+                image_content = build_image_message_content(task.file_path, task.question)
+
+                if not image_content:
+                    # 非图片文件：通过 attachments + files 注入 state
+                    attachments, files = build_attachments_and_files(
+                        task.file_path, task.file_name
+                    )
+
+            # 构建消息
+            if image_content:
+                # 图片：使用多模态 HumanMessage
+                messages = [HumanMessage(content=image_content)]
+            else:
+                messages = [HumanMessage(content=task.question)]
+
+            # 构建 graph 输入（包含 attachments 和 files）
+            graph_input = {"messages": messages}
+            if attachments:
+                graph_input["attachments"] = attachments
+            if files:
+                graph_input["files"] = files
+
+            # 直接调用 graph.ainvoke 以传递完整的初始 state
+            graph = await self._agent.get_graph()
+            context = self._agent.context_schema()
+            agent_config = input_context.get("agent_config")
+            if isinstance(agent_config, dict):
+                context.update(agent_config)
+            context.update(input_context)
+
+            invoke_config = {
+                "configurable": {
+                    "thread_id": context.thread_id,
+                    "user_id": context.user_id,
+                },
+                "recursion_limit": 100,
+            }
+
+            # 设置超时
+            result = await asyncio.wait_for(
+                graph.ainvoke(graph_input, context=context, config=invoke_config),
+                timeout=self.config.timeout,
+            )
+
+            predicted_answer = self._extract_answer(result)
+
+        except TimeoutError:
+            error = f"Timeout after {self.config.timeout}s"
+        except Exception as e:
+            error = f"{type(e).__name__}: {e}"
+
+        duration = time.time() - start_time
+        is_correct = GaiaScorer.score(predicted_answer, task.final_answer) if not error else False
+
+        return EvalResult(
+            task_id=task.task_id,
+            level=task.level,
+            question=task.question,
+            gold_answer=task.final_answer,
+            predicted_answer=predicted_answer,
+            is_correct=is_correct,
+            error=error,
+            duration_seconds=round(duration, 2),
+        )
+
+    async def run(self, tasks: list[GaiaTask]) -> list[EvalResult]:
+        """执行批量评估
+
+        Args:
+            tasks: 待评估的 GaiaTask 列表
+
+        Returns:
+            评估结果列表
+        """
+        await self._init_agent()
+
+        results: list[EvalResult] = []
+
+        if self.config.concurrency <= 1:
+            # 串行执行
+            for task in tqdm(tasks, desc="评估进度", unit="题"):
+                result = await self._run_single_task(task)
+                results.append(result)
+                status = "✓" if result.is_correct else ("✗" if not result.error else "⚠")
+                tqdm.write(
+                    f"  {status} [{result.task_id[:8]}] L{result.level} "
+                    f"| 预测: {result.predicted_answer[:50]!r} "
+                    f"| 标准: {result.gold_answer[:50]!r} "
+                    f"| {result.duration_seconds:.1f}s"
+                )
+        else:
+            # 并发执行（使用信号量控制并发度）
+            semaphore = asyncio.Semaphore(self.config.concurrency)
+            pbar = tqdm(total=len(tasks), desc="评估进度", unit="题")
+
+            async def bounded_run(task: GaiaTask) -> EvalResult:
+                async with semaphore:
+                    result = await self._run_single_task(task)
+                    pbar.update(1)
+                    return result
+
+            results = await asyncio.gather(*[bounded_run(t) for t in tasks])
+            pbar.close()
+
+        return results

test/gaia_eval/scorer.py

@@ -0,0 +1,97 @@
+"""GAIA 答案评分模块
+
+实现 GAIA 官方的 quasi exact match 评分逻辑。
+参考: https://huggingface.co/spaces/gaia-benchmark/leaderboard
+"""
+
+import re
+import string
+
+
+class GaiaScorer:
+    """GAIA 准精确匹配评分器"""
+
+    @staticmethod
+    def normalize_text(text: str) -> str:
+        """文本标准化
+
+        - 去除首尾空白
+        - 统一为小写
+        - 移除冠词 (a, an, the)
+        - 移除标点
+        - 合并连续空白
+        """
+        text = text.strip().lower()
+
+        # 移除冠词
+        text = re.sub(r"\b(a|an|the)\b", " ", text)
+
+        # 移除标点
+        text = text.translate(str.maketrans("", "", string.punctuation))
+
+        # 合并连续空白
+        text = re.sub(r"\s+", " ", text).strip()
+
+        return text
+
+    @staticmethod
+    def normalize_number(text: str) -> str:
+        """数字标准化
+
+        - "1,000" → "1000"
+        - "3.0" → "3"
+        - "$100" → "100"
+        - "100%" → "100"
+        """
+        # 去除货币符号
+        text = re.sub(r"[$€£¥]", "", text)
+        # 去除百分号
+        text = text.rstrip("%")
+        # 去除千分位逗号
+        text = text.replace(",", "")
+
+        # 判断是否为数字，若是则标准化
+        try:
+            num = float(text)
+            # 如果是整数则去掉 .0
+            if num == int(num):
+                return str(int(num))
+            return str(num)
+        except ValueError:
+            return text
+
+    @classmethod
+    def score(cls, prediction: str, gold: str) -> bool:
+        """计算单条评估的准精确匹配分数
+
+        Args:
+            prediction: 模型预测答案
+            gold: 黄金标准答案
+
+        Returns:
+            True 表示匹配，False 表示不匹配
+        """
+        if not prediction or not gold:
+            return False
+
+        # 先尝试数字比较
+        pred_num = cls.normalize_number(prediction.strip())
+        gold_num = cls.normalize_number(gold.strip())
+        try:
+            if float(pred_num) == float(gold_num):
+                return True
+        except ValueError:
+            pass
+
+        # 检查是否为列表答案（逗号分隔）
+        if "," in gold:
+            pred_items = sorted(cls.normalize_text(item) for item in prediction.split(","))
+            gold_items = sorted(cls.normalize_text(item) for item in gold.split(","))
+            if pred_items == gold_items:
+                return True
+
+        # 文本标准化比较
+        pred_normalized = cls.normalize_text(prediction)
+        gold_normalized = cls.normalize_text(gold)
+
+        return pred_normalized == gold_normalized

test/gaia_eval/test_scorer.py

@@ -0,0 +1,102 @@
+import sys
+from pathlib import Path
+
+import pytest
+
+# 确保可以找到 gaia_eval 包
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+from gaia_eval.scorer import GaiaScorer
+
+
+
+class TestNormalizeText:
+    """文本标准化测试"""
+
+    def test_basic(self):
+        assert GaiaScorer.normalize_text("  Hello World  ") == "hello world"
+
+    def test_remove_articles(self):
+        assert GaiaScorer.normalize_text("The quick brown fox") == "quick brown fox"
+        assert GaiaScorer.normalize_text("a cat") == "cat"
+        assert GaiaScorer.normalize_text("an apple") == "apple"
+
+    def test_remove_punctuation(self):
+        assert GaiaScorer.normalize_text("Hello, World!") == "hello world"
+        assert GaiaScorer.normalize_text("Dr. Smith's") == "dr smiths"
+
+    def test_collapse_whitespace(self):
+        assert GaiaScorer.normalize_text("hello   world") == "hello world"
+
+
+class TestNormalizeNumber:
+    """数字标准化测试"""
+
+    def test_remove_commas(self):
+        assert GaiaScorer.normalize_number("1,000") == "1000"
+        assert GaiaScorer.normalize_number("1,234,567") == "1234567"
+
+    def test_remove_trailing_zeros(self):
+        assert GaiaScorer.normalize_number("3.0") == "3"
+        assert GaiaScorer.normalize_number("100.00") == "100"
+
+    def test_keep_decimals(self):
+        assert GaiaScorer.normalize_number("3.14") == "3.14"
+
+    def test_currency(self):
+        assert GaiaScorer.normalize_number("$100") == "100"
+        assert GaiaScorer.normalize_number("€50") == "50"
+
+    def test_percentage(self):
+        assert GaiaScorer.normalize_number("75%") == "75"
+
+    def test_non_number(self):
+        assert GaiaScorer.normalize_number("hello") == "hello"
+
+
+class TestScore:
+    """评分逻辑测试"""
+
+    def test_exact_match(self):
+        assert GaiaScorer.score("Paris", "Paris") is True
+
+    def test_case_insensitive(self):
+        assert GaiaScorer.score("paris", "Paris") is True
+        assert GaiaScorer.score("PARIS", "paris") is True
+
+    def test_with_articles(self):
+        assert GaiaScorer.score("The Eiffel Tower", "Eiffel Tower") is True
+
+    def test_number_match(self):
+        assert GaiaScorer.score("1,000", "1000") is True
+        assert GaiaScorer.score("3.0", "3") is True
+        assert GaiaScorer.score("$100", "100") is True
+
+    def test_list_match(self):
+        assert GaiaScorer.score("a, b, c", "c, b, a") is True
+        assert GaiaScorer.score("Apple, Banana", "Banana, Apple") is True
+
+    def test_list_mismatch(self):
+        assert GaiaScorer.score("a, b", "a, b, c") is False
+
+    def test_mismatch(self):
+        assert GaiaScorer.score("London", "Paris") is False
+
+    def test_empty_prediction(self):
+        assert GaiaScorer.score("", "Paris") is False
+
+    def test_empty_gold(self):
+        assert GaiaScorer.score("Paris", "") is False
+
+    def test_both_empty(self):
+        assert GaiaScorer.score("", "") is False
+
+    def test_whitespace_handling(self):
+        assert GaiaScorer.score("  Paris  ", "Paris") is True
+
+    def test_number_with_text(self):
+        # 混合的数字和文本不应该触发数字匹配
+        assert GaiaScorer.score("42 years", "42") is False
+
+    def test_floating_point(self):
+        assert GaiaScorer.score("3.14", "3.14") is True
+        assert GaiaScorer.score("3.14", "3.15") is False