From 9c30c74d41e3cc814ca66fd2059cad496265ab12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=82=96=E6=B3=BD=E6=B6=9B?= Date: Sun, 1 Mar 2026 17:44:53 +0800 Subject: [PATCH] =?UTF-8?q?Revert=20"feat:=20=E9=9B=86=E6=88=90Langfuse?= =?UTF-8?q?=E5=B9=B6=E6=96=B0=E5=A2=9EGAIA=E7=A6=BB=E7=BA=BF=E8=AF=84?= =?UTF-8?q?=E4=BC=B0=E5=B7=A5=E5=85=B7"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 45ecf7bcf268e54579307ab2b5024e7406764b1e. --- docker-compose.yml | 2 - pyproject.toml | 5 - src/agents/common/base.py | 15 +- src/models/chat.py | 7 +- src/utils/langfuse_integration.py | 39 ----- test/gaia_eval/__init__.py | 21 --- test/gaia_eval/__main__.py | 109 ------------ test/gaia_eval/config.py | 57 ------- test/gaia_eval/dataset_loader.py | 164 ------------------ test/gaia_eval/file_handler.py | 191 --------------------- test/gaia_eval/reporter.py | 162 ------------------ test/gaia_eval/runner.py | 272 ------------------------------ test/gaia_eval/scorer.py | 97 ----------- test/gaia_eval/test_scorer.py | 102 ----------- uv.lock | 90 ---------- 15 files changed, 2 insertions(+), 1331 deletions(-) delete mode 100644 src/utils/langfuse_integration.py delete mode 100644 test/gaia_eval/__init__.py delete mode 100644 test/gaia_eval/__main__.py delete mode 100644 test/gaia_eval/config.py delete mode 100644 test/gaia_eval/dataset_loader.py delete mode 100644 test/gaia_eval/file_handler.py delete mode 100644 test/gaia_eval/reporter.py delete mode 100644 test/gaia_eval/runner.py delete mode 100644 test/gaia_eval/scorer.py delete mode 100644 test/gaia_eval/test_scorer.py diff --git a/docker-compose.yml b/docker-compose.yml index f7d399b2..ecc3e899 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,8 +13,6 @@ services: - ./test:/app/test - ./scripts:/app/scripts - ./.env:/app/.env - - ./pyproject.toml:/app/pyproject.toml - - ./uv.lock:/app/uv.lock - ${MODEL_DIR:-./models}:/models # 使用默认值处理未定义的环境变量 ports: - "5050:5050" diff --git a/pyproject.toml b/pyproject.toml index 2d13230a..3585f60c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,6 @@ dependencies = [ "torchvision==0.23", "docling>=2.68.0", "loguru>=0.7.3", - "langfuse>=2.0.0", "redis>=5.2.0", ] [tool.ruff] @@ -120,7 +119,3 @@ test = [ "pytest-httpx>=0.32.0", "pytest-cov>=6.0.0", ] -eval = [ - "datasets>=3.0.0", - "huggingface-hub>=0.25.0", -] diff --git a/src/agents/common/base.py b/src/agents/common/base.py index 48c86c1e..1504a2b0 100644 --- a/src/agents/common/base.py +++ b/src/agents/common/base.py @@ -14,7 +14,6 @@ from src import config as sys_config from src.agents.common.context import BaseContext from src.services.skill_resolver import get_skill_options_from_db from src.utils import logger -from src.utils.langfuse_integration import get_langfuse_callback class BaseAgent: @@ -76,13 +75,7 @@ class BaseAgent: if isinstance(agent_config, dict): context.update(agent_config) context.update(input_context or {}) - - stream_config = {} - langfuse_handler = get_langfuse_callback() - if langfuse_handler: - stream_config["callbacks"] = [langfuse_handler] - - for event in graph.astream({"messages": messages}, stream_mode="values", context=context, config=stream_config): + for event in graph.astream({"messages": messages}, stream_mode="values", context=context): yield event["messages"] async def stream_messages(self, messages: list[str], input_context=None, **kwargs): @@ -99,9 +92,6 @@ class BaseAgent: "configurable": {"thread_id": context.thread_id, "user_id": context.user_id}, "recursion_limit": 300, } - langfuse_handler = get_langfuse_callback() - if langfuse_handler: - input_config["callbacks"] = [langfuse_handler] async for msg, metadata in graph.astream( {"messages": messages}, @@ -125,9 +115,6 @@ class BaseAgent: "configurable": {"thread_id": context.thread_id, "user_id": context.user_id}, "recursion_limit": 100, } - langfuse_handler = get_langfuse_callback() - if langfuse_handler: - input_config["callbacks"] = [langfuse_handler] msg = await graph.ainvoke( {"messages": messages}, diff --git a/src/models/chat.py b/src/models/chat.py index 08de9309..27eda78a 100644 --- a/src/models/chat.py +++ b/src/models/chat.py @@ -1,16 +1,11 @@ import os import traceback +from openai import AsyncOpenAI from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_exponential from src import config from src.utils import logger -from src.utils.langfuse_integration import is_langfuse_enabled - -if is_langfuse_enabled(): - from langfuse.openai import AsyncOpenAI -else: - from openai import AsyncOpenAI def split_model_spec(model_spec, sep="/"): diff --git a/src/utils/langfuse_integration.py b/src/utils/langfuse_integration.py deleted file mode 100644 index e5238c44..00000000 --- a/src/utils/langfuse_integration.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Langfuse 集成模块 - -提供 Langfuse 的初始化和回调处理器获取功能。 -仅当环境变量 LANGFUSE_SECRET_KEY 和 LANGFUSE_PUBLIC_KEY 存在时启用。 -""" - -import os -from functools import lru_cache - -from src.utils.logging_config import logger - - -def is_langfuse_enabled() -> bool: - """检查 Langfuse 是否已配置""" - return bool( - os.getenv("LANGFUSE_SECRET_KEY") - and os.getenv("LANGFUSE_PUBLIC_KEY") - ) - - -@lru_cache(maxsize=1) -def get_langfuse_callback(): - """获取 Langfuse LangChain CallbackHandler（单例） - - Returns: - CallbackHandler 实例，若 Langfuse 未配置则返回 None - """ - if not is_langfuse_enabled(): - return None - try: - from langfuse.langchain import CallbackHandler - - handler = CallbackHandler() - logger.info("Langfuse CallbackHandler 初始化成功") - return handler - except Exception as e: - logger.warning(f"Langfuse CallbackHandler 初始化失败: {e}") - return None diff --git a/test/gaia_eval/__init__.py b/test/gaia_eval/__init__.py deleted file mode 100644 index 2dbfde58..00000000 --- a/test/gaia_eval/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -"""GAIA 离线评估框架 - -使用 GAIA benchmark 评估 Yuxi-Know 的 Agent 系统。 - -用法: - python -m test.gaia_eval --level 1 --max-samples 5 -""" - -from .config import EvalConfig -from .dataset_loader import GaiaDatasetLoader, GaiaTask -from .runner import EvalResult, GaiaEvalRunner -from .scorer import GaiaScorer - -__all__ = [ - "EvalConfig", - "GaiaDatasetLoader", - "GaiaTask", - "GaiaEvalRunner", - "EvalResult", - "GaiaScorer", -] diff --git a/test/gaia_eval/__main__.py b/test/gaia_eval/__main__.py deleted file mode 100644 index 238a017e..00000000 --- a/test/gaia_eval/__main__.py +++ /dev/null @@ -1,109 +0,0 @@ -"""GAIA 离线评估 CLI 入口 - -用法: - python -m test.gaia_eval # 评估全部级别 - python -m test.gaia_eval --level 1 # 只评估 Level 1 - python -m test.gaia_eval --level 1 --max-samples 5 # 限制 5 条（调试） - python -m test.gaia_eval --agent-config-id 3 # 使用数据库预设配置 - python -m test.gaia_eval --model deepseek-chat --level 1 # 指定模型 - python -m test.gaia_eval --output-dir ./my_results # 自定义输出目录 -""" - -import argparse -import asyncio -import sys - -from .config import EvalConfig -from .dataset_loader import GaiaDatasetLoader -from .reporter import GaiaReporter -from .runner import GaiaEvalRunner - - -def parse_args() -> EvalConfig: - """解析命令行参数""" - parser = argparse.ArgumentParser( - description="GAIA 离线评估 - 评估 Yuxi-Know Agent 系统", - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - - parser.add_argument("--agent-id", default="ChatbotAgent", help="Agent ID（默认: ChatbotAgent）") - parser.add_argument("--agent-config-id", type=int, default=None, help="Agent 预设配置 ID，从数据库加载") - parser.add_argument("--model", default=None, help="模型名称（当未指定 agent-config-id 时使用）") - parser.add_argument("--level", default="all", choices=["1", "2", "3", "all"], help="评估级别（默认: all）") - parser.add_argument("--split", default="validation", choices=["validation", "test"], help="数据集划分（默认: validation）") - parser.add_argument("--max-samples", type=int, default=None, help="最大评估样本数（调试用）") - parser.add_argument("--timeout", type=int, default=300, help="单题超时，秒（默认: 300）") - parser.add_argument("--concurrency", type=int, default=1, help="并发评估数（默认: 1）") - parser.add_argument("--output-dir", default="eval_results", help="结果输出目录（默认: eval_results）") - - args = parser.parse_args() - - from pathlib import Path - - return EvalConfig( - agent_id=args.agent_id, - agent_config_id=args.agent_config_id, - model=args.model, - level=args.level, - split=args.split, - max_samples=args.max_samples, - timeout=args.timeout, - concurrency=args.concurrency, - output_dir=Path(args.output_dir), - ) - - -async def main(): - """主流程""" - from rich.console import Console - - console = Console() - config = parse_args() - - # 1. 显示配置信息 - console.print("\n[bold cyan]═══ GAIA 离线评估 ═══[/bold cyan]") - console.print(f" Agent: {config.agent_id}") - if config.agent_config_id: - console.print(f" 配置ID: {config.agent_config_id}") - if config.model: - console.print(f" 模型: {config.model}") - console.print(f" 级别: {config.level}") - console.print(f" 数据集: {config.split}") - if config.max_samples: - console.print(f" 样本限制: {config.max_samples}") - console.print(f" 超时: {config.timeout}s") - console.print(f" 并发: {config.concurrency}") - console.print(f" 输出目录: {config.output_dir}") - console.print() - - # 2. 加载数据集 - console.print("[bold]正在加载 GAIA 数据集...[/bold]") - try: - loader = GaiaDatasetLoader(config) - tasks = loader.load_tasks() - except Exception as e: - console.print(f"[red]数据集加载失败: {e}[/red]") - console.print("[dim]请确保已设置 HF_TOKEN 并同意了数据集使用条款[/dim]") - sys.exit(1) - - console.print(f" 已加载 {len(tasks)} 条评估任务\n") - - if not tasks: - console.print("[yellow]没有可评估的任务[/yellow]") - sys.exit(0) - - # 3. 执行评估 - console.print("[bold]开始评估...[/bold]\n") - runner = GaiaEvalRunner(config) - results = await runner.run(tasks) - - # 4. 生成报告 - reporter = GaiaReporter(config, results) - reporter.print_report() - - report_path = reporter.save_json_report() - console.print(f"[green]详细报告已保存: {report_path}[/green]\n") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/test/gaia_eval/config.py b/test/gaia_eval/config.py deleted file mode 100644 index 817647de..00000000 --- a/test/gaia_eval/config.py +++ /dev/null @@ -1,57 +0,0 @@ -"""GAIA 评估配置管理模块""" - -from dataclasses import dataclass, field -from pathlib import Path - - -@dataclass -class EvalConfig: - """GAIA 评估配置 - - 配置优先级：CLI 参数 > 环境变量 > 默认值 - 当指定 agent_config_id 时，会从数据库加载预设配置（model/tools/knowledges/system_prompt 等） - """ - - # Agent 相关 - agent_id: str = "ChatbotAgent" - agent_config_id: int | None = None # 从数据库加载预设配置 - - # 备选配置（当未指定 agent_config_id 时使用） - model: str | None = None - system_prompt: str | None = None - tools: list[dict] | None = None - knowledges: list[str] | None = None - - # 数据集相关 - level: str = "all" # "1", "2", "3", "all" - split: str = "validation" # "validation" or "test" - max_samples: int | None = None # 限制评估样本数（调试用） - - # 运行控制 - timeout: int = 300 # 单题超时，秒 - concurrency: int = 1 # 并发评估数 - - # 输出相关 - output_dir: Path = field(default_factory=lambda: Path("eval_results")) - - def get_dataset_configs(self) -> list[str]: - """根据 level 获取 HuggingFace dataset config 名称列表""" - if self.level == "all": - return ["2023_level1", "2023_level2", "2023_level3"] - return [f"2023_level{self.level}"] - - def build_agent_config(self) -> dict: - """构建不使用 agent_config_id 时的 agent_config 字典 - - 当未指定 agent_config_id 时，从 CLI 参数组装配置。 - """ - config = {} - if self.model: - config["model"] = self.model - if self.system_prompt: - config["system_prompt"] = self.system_prompt - if self.tools is not None: - config["tools"] = self.tools - if self.knowledges is not None: - config["knowledges"] = self.knowledges - return config diff --git a/test/gaia_eval/dataset_loader.py b/test/gaia_eval/dataset_loader.py deleted file mode 100644 index 8e3ec7c6..00000000 --- a/test/gaia_eval/dataset_loader.py +++ /dev/null @@ -1,164 +0,0 @@ -"""GAIA 数据集加载器 - -支持两种模式： - 1. 流式模式（默认）：直接 streaming 拉取 metadata，无需完整下载，适合快速测试 - 2. 完整模式：snapshot_download 后加载，附件文件可本地读取 - -需要设置 HF_TOKEN 环境变量，并在 HuggingFace 上同意数据集使用条款。 -""" - -import os -from dataclasses import dataclass - -from .config import EvalConfig - - -@dataclass -class GaiaTask: - """单条 GAIA 评估任务""" - - task_id: str - question: str - level: int - final_answer: str - file_name: str | None = None - file_path: str | None = None # 流式模式下为 None（无本地文件） - annotator_metadata: dict | None = None - - -def _get_token() -> str | None: - """获取 HF_TOKEN，支持从环境变量或 .env 文件读取""" - token = os.environ.get("HF_TOKEN") - if not token: - try: - from dotenv import load_dotenv - load_dotenv() - token = os.environ.get("HF_TOKEN") - except ImportError: - pass - return token - - -class GaiaDatasetLoader: - """GAIA 数据集加载器""" - - REPO_ID = "gaia-benchmark/GAIA" - - def __init__(self, config: EvalConfig): - self.config = config - self._data_dir: str | None = None # 完整下载后的本地路径 - - def load_tasks(self) -> list[GaiaTask]: - """加载并返回 GaiaTask 列表 - - 当 max_samples 较小时，自动使用流式加载避免完整下载； - 否则使用 snapshot_download 完整下载（支持附件文件读取）。 - """ - # 小样本时用流式加载，避免下载完整数据集 - small_sample = self.config.max_samples and self.config.max_samples <= 50 - if small_sample: - return self._load_streaming() - else: - return self._load_full() - - def _load_streaming(self) -> list[GaiaTask]: - """流式加载：直接从 HuggingFace 流式拉取，附件文件按需单独下载""" - from datasets import load_dataset - from huggingface_hub import hf_hub_download - - token = _get_token() - if not token: - raise RuntimeError( - "未设置 HF_TOKEN 环境变量。请在 .env 中添加 HF_TOKEN=hf_xxx" - ) - - dataset_configs = self.config.get_dataset_configs() - tasks: list[GaiaTask] = [] - - for ds_config in dataset_configs: - dataset = load_dataset( - self.REPO_ID, - ds_config, - split=self.config.split, - streaming=True, - token=token, - ) - - for example in dataset: - file_name = example.get("file_name") or None - file_path = None - - # 按需下载单个附件文件（只下载当前任务的文件，不下载整个数据集） - if file_name: - repo_file_path = f"2023/{self.config.split}/{file_name}" - try: - file_path = hf_hub_download( - repo_id=self.REPO_ID, - repo_type="dataset", - filename=repo_file_path, - token=token, - ) - except Exception as e: - import warnings - warnings.warn(f"附件下载失败 {file_name}: {e}") - - task = GaiaTask( - task_id=example["task_id"], - question=example["Question"], - level=int(example["Level"]), - final_answer=example.get("Final answer", ""), - file_name=file_name, - file_path=file_path, - annotator_metadata=example.get("Annotator Metadata"), - ) - tasks.append(task) - - if self.config.max_samples and len(tasks) >= self.config.max_samples: - return tasks - - return tasks - - def _load_full(self) -> list[GaiaTask]: - """完整下载：先 snapshot_download，再加载（附件文件可本地读取）""" - from datasets import load_dataset - from huggingface_hub import snapshot_download - - token = _get_token() - if not token: - raise RuntimeError( - "未设置 HF_TOKEN 环境变量。请在 .env 中添加 HF_TOKEN=hf_xxx" - ) - - if not self._data_dir: - self._data_dir = snapshot_download( - repo_id=self.REPO_ID, - repo_type="dataset", - token=token, - ) - - dataset_configs = self.config.get_dataset_configs() - tasks: list[GaiaTask] = [] - - for ds_config in dataset_configs: - dataset = load_dataset(self._data_dir, ds_config, split=self.config.split) - - for example in dataset: - file_path = example.get("file_path") - if file_path: - file_path = os.path.join(self._data_dir, file_path) - - task = GaiaTask( - task_id=example["task_id"], - question=example["Question"], - level=int(example["Level"]), - final_answer=example.get("Final answer", ""), - file_name=example.get("file_name") or None, - file_path=file_path, - annotator_metadata=example.get("Annotator Metadata"), - ) - tasks.append(task) - - if self.config.max_samples and len(tasks) > self.config.max_samples: - tasks = tasks[: self.config.max_samples] - - return tasks diff --git a/test/gaia_eval/file_handler.py b/test/gaia_eval/file_handler.py deleted file mode 100644 index 9f526322..00000000 --- a/test/gaia_eval/file_handler.py +++ /dev/null @@ -1,191 +0,0 @@ -"""GAIA 评估文件处理模块 - -将 GAIA 数据集中的附件文件转换为 Agent 可识别的 attachments + state files 格式。 -""" - -from __future__ import annotations - -import base64 -import mimetypes -import os -from datetime import datetime - -from src.utils import logger - - -# 支持直接读取内容的文本类文件扩展名 -TEXT_EXTENSIONS = { - ".txt", ".md", ".csv", ".tsv", ".json", ".jsonl", - ".xml", ".html", ".htm", ".yaml", ".yml", ".log", - ".py", ".js", ".ts", ".java", ".c", ".cpp", ".h", - ".rb", ".go", ".rs", ".sh", ".bat", ".sql", - ".ini", ".cfg", ".conf", ".toml", -} - -# 支持用 PDF 解析的扩展名 -PDF_EXTENSIONS = {".pdf"} - -# 支持用 pandas 等读取的表格扩展名 -SPREADSHEET_EXTENSIONS = {".xlsx", ".xls"} - -# 图片类扩展名（用于多模态消息） -IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp"} - - -def _read_text_file(file_path: str) -> str | None: - """读取纯文本文件""" - encodings = ["utf-8", "latin-1", "gbk"] - for enc in encodings: - try: - with open(file_path, encoding=enc) as f: - return f.read() - except (UnicodeDecodeError, OSError): - continue - return None - - -def _read_pdf_file(file_path: str) -> str | None: - """解析 PDF 文件为文本""" - try: - import fitz # PyMuPDF - - doc = fitz.open(file_path) - pages = [] - for page in doc: - pages.append(page.get_text()) - doc.close() - return "\n\n".join(pages) - except ImportError: - logger.warning("PyMuPDF (fitz) 未安装，无法解析 PDF。请安装: pip install PyMuPDF") - return None - except Exception as e: - logger.warning(f"PDF 解析失败 {file_path}: {e}") - return None - - -def _read_spreadsheet_file(file_path: str) -> str | None: - """读取 Excel 文件并转为 CSV 文本""" - try: - import pandas as pd - - df = pd.read_excel(file_path) - return df.to_csv(index=False) - except ImportError: - logger.warning("pandas/openpyxl 未安装，无法解析 Excel。请安装: pip install pandas openpyxl") - return None - except Exception as e: - logger.warning(f"Excel 解析失败 {file_path}: {e}") - return None - - -def _read_image_as_base64(file_path: str) -> str | None: - """读取图片为 base64""" - try: - with open(file_path, "rb") as f: - data = f.read() - return base64.b64encode(data).decode("utf-8") - except Exception as e: - logger.warning(f"图片读取失败 {file_path}: {e}") - return None - - -def read_file_content(file_path: str) -> str | None: - """根据文件类型读取内容，返回文本""" - if not file_path or not os.path.exists(file_path): - logger.warning(f"文件不存在: {file_path}") - return None - - _, ext = os.path.splitext(file_path) - ext = ext.lower() - - if ext in TEXT_EXTENSIONS: - return _read_text_file(file_path) - elif ext in PDF_EXTENSIONS: - return _read_pdf_file(file_path) - elif ext in SPREADSHEET_EXTENSIONS: - return _read_spreadsheet_file(file_path) - else: - # 尝试作为文本读取 - content = _read_text_file(file_path) - if content: - return content - logger.warning(f"不支持的文件类型: {ext} ({file_path})") - return None - - -def build_attachments_and_files( - file_path: str, - file_name: str | None = None, -) -> tuple[list[dict], dict]: - """将文件转换为 Agent 的 attachments 和 state files 格式 - - 返回: - (attachments, files) 元组： - - attachments: 附件元数据列表，用于 AttachmentMiddleware - - files: StateBackend 格式的文件字典，用于 read_file 工具 - """ - if not file_path or not os.path.exists(file_path): - return [], {} - - if not file_name: - file_name = os.path.basename(file_path) - - _, ext = os.path.splitext(file_path) - ext = ext.lower() - - # 图片使用多模态消息，不走 attachment 机制 - if ext in IMAGE_EXTENSIONS: - return [], {} - - content = read_file_content(file_path) - if not content: - return [], {} - - now = datetime.utcnow().isoformat() + "+00:00" - virtual_path = f"/attachments/{file_name}" - - # 构建 attachment 元数据 - attachment = { - "file_name": file_name, - "file_path": virtual_path, - "status": "parsed", - "markdown": content, - "uploaded_at": now, - "truncated": False, - } - - # 构建 state files（与 _build_state_files 格式一致） - content_lines = content.split("\n") - files = { - virtual_path: { - "content": content_lines, - "created_at": now, - "modified_at": now, - } - } - - return [attachment], files - - -def build_image_message_content(file_path: str, question: str) -> list[dict] | None: - """为图片文件构建多模态消息内容 - - 返回 HumanMessage 的 content（多模态格式），或 None 如果不是图片。 - """ - if not file_path or not os.path.exists(file_path): - return None - - _, ext = os.path.splitext(file_path) - if ext.lower() not in IMAGE_EXTENSIONS: - return None - - base64_data = _read_image_as_base64(file_path) - if not base64_data: - return None - - mime_type = mimetypes.guess_type(file_path)[0] or "image/jpeg" - - return [ - {"type": "text", "text": question}, - {"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{base64_data}"}}, - ] diff --git a/test/gaia_eval/reporter.py b/test/gaia_eval/reporter.py deleted file mode 100644 index 544c16fa..00000000 --- a/test/gaia_eval/reporter.py +++ /dev/null @@ -1,162 +0,0 @@ -"""GAIA 评估报告生成模块 - -输出终端彩色表格和 JSON 详细报告。 -""" - -import json -from collections import defaultdict -from datetime import datetime -from pathlib import Path - -from .config import EvalConfig -from .runner import EvalResult - - -class GaiaReporter: - """评估报告生成器""" - - def __init__(self, config: EvalConfig, results: list[EvalResult]): - self.config = config - self.results = results - - def _compute_stats(self) -> dict: - """计算统计数据""" - total = len(self.results) - correct = sum(1 for r in self.results if r.is_correct) - errors = sum(1 for r in self.results if r.error is not None) - - # 按 Level 分组统计 - by_level: dict[int, dict] = defaultdict(lambda: {"total": 0, "correct": 0, "errors": 0}) - for r in self.results: - by_level[r.level]["total"] += 1 - if r.is_correct: - by_level[r.level]["correct"] += 1 - if r.error: - by_level[r.level]["errors"] += 1 - - # 耗时统计 - durations = [r.duration_seconds for r in self.results if not r.error] - avg_duration = sum(durations) / len(durations) if durations else 0 - max_duration = max(durations) if durations else 0 - min_duration = min(durations) if durations else 0 - - return { - "total": total, - "correct": correct, - "accuracy": round(correct / total * 100, 2) if total > 0 else 0, - "errors": errors, - "by_level": { - level: { - "total": stats["total"], - "correct": stats["correct"], - "accuracy": round(stats["correct"] / stats["total"] * 100, 2) if stats["total"] > 0 else 0, - "errors": stats["errors"], - } - for level, stats in sorted(by_level.items()) - }, - "duration": { - "avg": round(avg_duration, 2), - "max": round(max_duration, 2), - "min": round(min_duration, 2), - "total": round(sum(durations), 2), - }, - } - - def print_report(self): - """在终端输出彩色报告""" - from rich.console import Console - from rich.table import Table - - console = Console() - stats = self._compute_stats() - - # 标题 - console.print("\n[bold cyan]═══ GAIA 评估报告 ═══[/bold cyan]\n") - - # 总体统计 - accuracy_color = "green" if stats["accuracy"] >= 50 else ("yellow" if stats["accuracy"] >= 25 else "red") - console.print(f" Agent: [bold]{self.config.agent_id}[/bold]") - if self.config.agent_config_id: - console.print(f" 配置ID: [bold]{self.config.agent_config_id}[/bold]") - console.print(f" 总题数: {stats['total']}") - console.print(f" 正确数: {stats['correct']}") - console.print(f" 错误数: {stats['errors']}") - console.print(f" 准确率: [{accuracy_color}]{stats['accuracy']}%[/{accuracy_color}]") - console.print() - - # 按 Level 统计表 - level_table = Table(title="按 Level 统计") - level_table.add_column("Level", style="bold") - level_table.add_column("总数", justify="right") - level_table.add_column("正确", justify="right") - level_table.add_column("准确率", justify="right") - level_table.add_column("错误", justify="right") - - for level, level_stats in stats["by_level"].items(): - acc_color = "green" if level_stats["accuracy"] >= 50 else ( - "yellow" if level_stats["accuracy"] >= 25 else "red" - ) - level_table.add_row( - f"Level {level}", - str(level_stats["total"]), - str(level_stats["correct"]), - f"[{acc_color}]{level_stats['accuracy']}%[/{acc_color}]", - str(level_stats["errors"]), - ) - - console.print(level_table) - console.print() - - # 耗时统计 - console.print("[bold]耗时统计:[/bold]") - console.print(f" 平均: {stats['duration']['avg']}s") - console.print(f" 最大: {stats['duration']['max']}s") - console.print(f" 最小: {stats['duration']['min']}s") - console.print(f" 总计: {stats['duration']['total']}s") - console.print() - - # 错误样本 - error_results = [r for r in self.results if r.error] - if error_results: - console.print(f"[bold red]错误样本 ({len(error_results)} 条):[/bold red]") - for r in error_results[:5]: - console.print(f" • [{r.task_id[:8]}] L{r.level}: {r.error}") - if len(error_results) > 5: - console.print(f" ... 还有 {len(error_results) - 5} 条错误") - console.print() - - def save_json_report(self) -> Path: - """保存 JSON 详细报告""" - self.config.output_dir.mkdir(parents=True, exist_ok=True) - - stats = self._compute_stats() - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - report_path = self.config.output_dir / f"gaia_eval_{timestamp}.json" - - report = { - "metadata": { - "timestamp": timestamp, - "agent_id": self.config.agent_id, - "agent_config_id": self.config.agent_config_id, - "level": self.config.level, - "split": self.config.split, - "timeout": self.config.timeout, - }, - "summary": stats, - "results": [ - { - "task_id": r.task_id, - "level": r.level, - "question": r.question, - "gold_answer": r.gold_answer, - "predicted_answer": r.predicted_answer, - "is_correct": r.is_correct, - "error": r.error, - "duration_seconds": r.duration_seconds, - } - for r in self.results - ], - } - - report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") - return report_path diff --git a/test/gaia_eval/runner.py b/test/gaia_eval/runner.py deleted file mode 100644 index 05b951bd..00000000 --- a/test/gaia_eval/runner.py +++ /dev/null @@ -1,272 +0,0 @@ -"""GAIA 评估运行器 - -核心评估逻辑：加载 agent、执行评估、收集结果。 -""" - -import asyncio -import time -import uuid -from dataclasses import dataclass - -from tqdm import tqdm - -from .config import EvalConfig -from .dataset_loader import GaiaTask -from .scorer import GaiaScorer - - -@dataclass -class EvalResult: - """单条评估结果""" - - task_id: str - level: int - question: str - gold_answer: str - predicted_answer: str - is_correct: bool - error: str | None = None - duration_seconds: float = 0.0 - - -class GaiaEvalRunner: - """GAIA 评估运行器 - - 通过直接调用 BaseAgent.invoke_messages() 执行评估， - 无需启动 Web 服务。 - """ - - def __init__(self, config: EvalConfig): - self.config = config - self._agent = None - self._agent_config: dict = {} - - async def _init_agent(self): - """初始化 Agent 实例并加载预设配置""" - from src.agents import agent_manager - - self._agent = agent_manager.get_agent(self.config.agent_id) - - # 如果指定了 agent_config_id，从数据库加载预设配置 - if self.config.agent_config_id is not None: - await self._load_agent_config_from_db() - else: - # 使用 CLI 参数组装配置 - self._agent_config = self.config.build_agent_config() - - async def _load_agent_config_from_db(self): - """从数据库加载 agent 预设配置""" - from src.repositories.agent_config_repository import AgentConfigRepository - from src.storage.postgres.manager import pg_manager - - # 评估脚本独立运行时，pg_manager 和 mcp_service 未经过 FastAPI 启动事件初始化，需手动初始化 - if not pg_manager._initialized: - pg_manager.initialize() - - from src.services.mcp_service import init_mcp_servers - await init_mcp_servers() - - async with pg_manager.get_async_session_context() as db: - repo = AgentConfigRepository(db) - config_item = await repo.get_by_id(self.config.agent_config_id) - - if config_item is None: - raise ValueError(f"Agent 配置 ID {self.config.agent_config_id} 不存在") - - # 与线上逻辑保持一致：从 config_json.context 中提取配置 - self._agent_config = (config_item.config_json or {}).get("context", {}) - - def _extract_answer(self, result: dict) -> str: - """从 agent 返回结果中提取最终答案 - - Agent 返回格式为 { "messages": [...] }，取最后一条 AI 消息的内容。 - """ - messages = result.get("messages", []) - if not messages: - return "" - - # 从后往前找第一条 AI 消息 - for msg in reversed(messages): - msg_type = getattr(msg, "type", None) - if msg_type == "ai": - content = getattr(msg, "content", "") - if isinstance(content, str) and content.strip(): - return self._parse_final_answer(content) - - return "" - - @staticmethod - def _parse_final_answer(content: str) -> str: - """尝试从 AI 回复中解析出最终答案 - - 如果回复中包含 "FINAL ANSWER:" 标记，则提取标记后面的内容。 - 否则返回完整回复。 - """ - # 常见答案标记格式 - markers = [ - "FINAL ANSWER:", - "Final Answer:", - "final answer:", - "最终答案：", - "最终答案:", - "答案：", - "答案:", - ] - for marker in markers: - if marker in content: - answer = content.split(marker, 1)[1].strip() - # 取第一行（去掉后续的解释） - first_line = answer.split("\n")[0].strip() - return first_line - - # 如果只有一行，直接返回 - lines = [ln.strip() for ln in content.strip().split("\n") if ln.strip()] - if len(lines) == 1: - return lines[0] - - # 多行回复，返回最后一行（通常是最终答案） - return lines[-1] if lines else content.strip() - - async def _run_single_task(self, task: GaiaTask) -> EvalResult: - """执行单条评估任务""" - start_time = time.time() - predicted_answer = "" - error = None - - try: - from langchain.messages import HumanMessage - - from .file_handler import build_attachments_and_files, build_image_message_content - - # 构建系统提示词 - gaia_system_suffix = ( - "\n\nIMPORTANT: When you have found the answer, " - "state it clearly on a single line prefixed with 'FINAL ANSWER: '. " - "Keep the answer as concise as possible - just the answer itself, " - "no extra explanation." - ) - - input_context = { - "user_id": "gaia_eval", - "thread_id": str(uuid.uuid4()), - "agent_config": { - **self._agent_config, - "system_prompt": ( - self._agent_config.get("system_prompt", "You are a helpful assistant.") - + gaia_system_suffix - ), - }, - } - - # 处理附件文件 - attachments: list[dict] = [] - files: dict = {} - image_content: list[dict] | None = None - - if task.file_path: - # 尝试构建图片多模态消息 - image_content = build_image_message_content(task.file_path, task.question) - - if not image_content: - # 非图片文件：通过 attachments + files 注入 state - attachments, files = build_attachments_and_files( - task.file_path, task.file_name - ) - - # 构建消息 - if image_content: - # 图片：使用多模态 HumanMessage - messages = [HumanMessage(content=image_content)] - else: - messages = [HumanMessage(content=task.question)] - - # 构建 graph 输入（包含 attachments 和 files） - graph_input = {"messages": messages} - if attachments: - graph_input["attachments"] = attachments - if files: - graph_input["files"] = files - - # 直接调用 graph.ainvoke 以传递完整的初始 state - graph = await self._agent.get_graph() - context = self._agent.context_schema() - agent_config = input_context.get("agent_config") - if isinstance(agent_config, dict): - context.update(agent_config) - context.update(input_context) - - invoke_config = { - "configurable": { - "thread_id": context.thread_id, - "user_id": context.user_id, - }, - "recursion_limit": 100, - } - - # 设置超时 - result = await asyncio.wait_for( - graph.ainvoke(graph_input, context=context, config=invoke_config), - timeout=self.config.timeout, - ) - - predicted_answer = self._extract_answer(result) - - except TimeoutError: - error = f"Timeout after {self.config.timeout}s" - except Exception as e: - error = f"{type(e).__name__}: {e}" - - duration = time.time() - start_time - is_correct = GaiaScorer.score(predicted_answer, task.final_answer) if not error else False - - return EvalResult( - task_id=task.task_id, - level=task.level, - question=task.question, - gold_answer=task.final_answer, - predicted_answer=predicted_answer, - is_correct=is_correct, - error=error, - duration_seconds=round(duration, 2), - ) - - async def run(self, tasks: list[GaiaTask]) -> list[EvalResult]: - """执行批量评估 - - Args: - tasks: 待评估的 GaiaTask 列表 - - Returns: - 评估结果列表 - """ - await self._init_agent() - - results: list[EvalResult] = [] - - if self.config.concurrency <= 1: - # 串行执行 - for task in tqdm(tasks, desc="评估进度", unit="题"): - result = await self._run_single_task(task) - results.append(result) - status = "✓" if result.is_correct else ("✗" if not result.error else "⚠") - tqdm.write( - f" {status} [{result.task_id[:8]}] L{result.level} " - f"| 预测: {result.predicted_answer[:50]!r} " - f"| 标准: {result.gold_answer[:50]!r} " - f"| {result.duration_seconds:.1f}s" - ) - else: - # 并发执行（使用信号量控制并发度） - semaphore = asyncio.Semaphore(self.config.concurrency) - pbar = tqdm(total=len(tasks), desc="评估进度", unit="题") - - async def bounded_run(task: GaiaTask) -> EvalResult: - async with semaphore: - result = await self._run_single_task(task) - pbar.update(1) - return result - - results = await asyncio.gather(*[bounded_run(t) for t in tasks]) - pbar.close() - - return results diff --git a/test/gaia_eval/scorer.py b/test/gaia_eval/scorer.py deleted file mode 100644 index ba4edfdd..00000000 --- a/test/gaia_eval/scorer.py +++ /dev/null @@ -1,97 +0,0 @@ -"""GAIA 答案评分模块 - -实现 GAIA 官方的 quasi exact match 评分逻辑。 -参考: https://huggingface.co/spaces/gaia-benchmark/leaderboard -""" - -import re -import string - - -class GaiaScorer: - """GAIA 准精确匹配评分器""" - - @staticmethod - def normalize_text(text: str) -> str: - """文本标准化 - - - 去除首尾空白 - - 统一为小写 - - 移除冠词 (a, an, the) - - 移除标点 - - 合并连续空白 - """ - text = text.strip().lower() - - # 移除冠词 - text = re.sub(r"\b(a|an|the)\b", " ", text) - - # 移除标点 - text = text.translate(str.maketrans("", "", string.punctuation)) - - # 合并连续空白 - text = re.sub(r"\s+", " ", text).strip() - - return text - - @staticmethod - def normalize_number(text: str) -> str: - """数字标准化 - - - "1,000" → "1000" - - "3.0" → "3" - - "$100" → "100" - - "100%" → "100" - """ - # 去除货币符号 - text = re.sub(r"[$€£¥]", "", text) - # 去除百分号 - text = text.rstrip("%") - # 去除千分位逗号 - text = text.replace(",", "") - - # 判断是否为数字，若是则标准化 - try: - num = float(text) - # 如果是整数则去掉 .0 - if num == int(num): - return str(int(num)) - return str(num) - except ValueError: - return text - - @classmethod - def score(cls, prediction: str, gold: str) -> bool: - """计算单条评估的准精确匹配分数 - - Args: - prediction: 模型预测答案 - gold: 黄金标准答案 - - Returns: - True 表示匹配，False 表示不匹配 - """ - if not prediction or not gold: - return False - - # 先尝试数字比较 - pred_num = cls.normalize_number(prediction.strip()) - gold_num = cls.normalize_number(gold.strip()) - try: - if float(pred_num) == float(gold_num): - return True - except ValueError: - pass - - # 检查是否为列表答案（逗号分隔） - if "," in gold: - pred_items = sorted(cls.normalize_text(item) for item in prediction.split(",")) - gold_items = sorted(cls.normalize_text(item) for item in gold.split(",")) - if pred_items == gold_items: - return True - - # 文本标准化比较 - pred_normalized = cls.normalize_text(prediction) - gold_normalized = cls.normalize_text(gold) - - return pred_normalized == gold_normalized diff --git a/test/gaia_eval/test_scorer.py b/test/gaia_eval/test_scorer.py deleted file mode 100644 index 4bb283a8..00000000 --- a/test/gaia_eval/test_scorer.py +++ /dev/null @@ -1,102 +0,0 @@ -import sys -from pathlib import Path - -import pytest - -# 确保可以找到 gaia_eval 包 -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -from gaia_eval.scorer import GaiaScorer - - - -class TestNormalizeText: - """文本标准化测试""" - - def test_basic(self): - assert GaiaScorer.normalize_text(" Hello World ") == "hello world" - - def test_remove_articles(self): - assert GaiaScorer.normalize_text("The quick brown fox") == "quick brown fox" - assert GaiaScorer.normalize_text("a cat") == "cat" - assert GaiaScorer.normalize_text("an apple") == "apple" - - def test_remove_punctuation(self): - assert GaiaScorer.normalize_text("Hello, World!") == "hello world" - assert GaiaScorer.normalize_text("Dr. Smith's") == "dr smiths" - - def test_collapse_whitespace(self): - assert GaiaScorer.normalize_text("hello world") == "hello world" - - -class TestNormalizeNumber: - """数字标准化测试""" - - def test_remove_commas(self): - assert GaiaScorer.normalize_number("1,000") == "1000" - assert GaiaScorer.normalize_number("1,234,567") == "1234567" - - def test_remove_trailing_zeros(self): - assert GaiaScorer.normalize_number("3.0") == "3" - assert GaiaScorer.normalize_number("100.00") == "100" - - def test_keep_decimals(self): - assert GaiaScorer.normalize_number("3.14") == "3.14" - - def test_currency(self): - assert GaiaScorer.normalize_number("$100") == "100" - assert GaiaScorer.normalize_number("€50") == "50" - - def test_percentage(self): - assert GaiaScorer.normalize_number("75%") == "75" - - def test_non_number(self): - assert GaiaScorer.normalize_number("hello") == "hello" - - -class TestScore: - """评分逻辑测试""" - - def test_exact_match(self): - assert GaiaScorer.score("Paris", "Paris") is True - - def test_case_insensitive(self): - assert GaiaScorer.score("paris", "Paris") is True - assert GaiaScorer.score("PARIS", "paris") is True - - def test_with_articles(self): - assert GaiaScorer.score("The Eiffel Tower", "Eiffel Tower") is True - - def test_number_match(self): - assert GaiaScorer.score("1,000", "1000") is True - assert GaiaScorer.score("3.0", "3") is True - assert GaiaScorer.score("$100", "100") is True - - def test_list_match(self): - assert GaiaScorer.score("a, b, c", "c, b, a") is True - assert GaiaScorer.score("Apple, Banana", "Banana, Apple") is True - - def test_list_mismatch(self): - assert GaiaScorer.score("a, b", "a, b, c") is False - - def test_mismatch(self): - assert GaiaScorer.score("London", "Paris") is False - - def test_empty_prediction(self): - assert GaiaScorer.score("", "Paris") is False - - def test_empty_gold(self): - assert GaiaScorer.score("Paris", "") is False - - def test_both_empty(self): - assert GaiaScorer.score("", "") is False - - def test_whitespace_handling(self): - assert GaiaScorer.score(" Paris ", "Paris") is True - - def test_number_with_text(self): - # 混合的数字和文本不应该触发数字匹配 - assert GaiaScorer.score("42 years", "42") is False - - def test_floating_point(self): - assert GaiaScorer.score("3.14", "3.14") is True - assert GaiaScorer.score("3.14", "3.15") is False diff --git a/uv.lock b/uv.lock index 0f095389..ae1ee104 100644 --- a/uv.lock +++ b/uv.lock @@ -611,31 +611,6 @@ wheels = [ { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686, upload-time = "2024-06-09T16:20:16.715Z" }, ] -[[package]] -name = "datasets" -version = "4.5.0" -source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } -dependencies = [ - { name = "dill" }, - { name = "filelock" }, - { name = "fsspec", extra = ["http"] }, - { name = "httpx" }, - { name = "huggingface-hub" }, - { name = "multiprocess" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pandas" }, - { name = "pyarrow" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "tqdm" }, - { name = "xxhash" }, -] -sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/55/bf/bb927bde63d649296c83e883171ae77074717c1b80fe2868b328bd0dbcbb/datasets-4.5.0.tar.gz", hash = "sha256:00c698ce1c2452e646cc5fad47fef39d3fe78dd650a8a6eb205bb45eb63cd500", size = 588384, upload-time = "2026-01-14T18:27:54.297Z" } -wheels = [ - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fc/d5/0d563ea3c205eee226dc8053cf7682a8ac588db8acecd0eda2b587987a0b/datasets-4.5.0-py3-none-any.whl", hash = "sha256:b5d7e08096ffa407dd69e58b1c0271c9b2506140839b8d99af07375ad31b6726", size = 515196, upload-time = "2026-01-14T18:27:52.419Z" }, -] - [[package]] name = "deepagents" version = "0.3.6" @@ -993,11 +968,6 @@ wheels = [ { url = "https://pypi.tuna.tsinghua.edu.cn/packages/eb/02/a6b21098b1d5d6249b7c5ab69dde30108a71e4e819d4a9778f1de1d5b70d/fsspec-2025.10.0-py3-none-any.whl", hash = "sha256:7c7712353ae7d875407f97715f0e1ffcc21e33d5b24556cb1e090ae9409ec61d", size = 200966, upload-time = "2025-10-30T14:58:42.53Z" }, ] -[package.optional-dependencies] -http = [ - { name = "aiohttp" }, -] - [[package]] name = "google-api-core" version = "2.29.0" @@ -1720,27 +1690,6 @@ dependencies = [ ] sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0e/72/a3add0e4eec4eb9e2569554f7c70f4a3c27712f40e3284d483e88094cc0e/langdetect-1.0.9.tar.gz", hash = "sha256:cbc1fef89f8d062739774bd51eda3da3274006b3661d199c2655f6b3f6d605a0", size = 981474, upload-time = "2021-05-07T07:54:13.562Z" } -[[package]] -name = "langfuse" -version = "3.14.5" -source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } -dependencies = [ - { name = "backoff" }, - { name = "httpx" }, - { name = "openai" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-http" }, - { name = "opentelemetry-sdk" }, - { name = "packaging" }, - { name = "pydantic" }, - { name = "requests" }, - { name = "wrapt" }, -] -sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ec/6b/7a945e8bc56cbf343b6f6171fd45870b0ea80ea38463b2db8dd5a9dc04a2/langfuse-3.14.5.tar.gz", hash = "sha256:2f543ec1540053d39b08a50ed5992caf1cd54d472a55cb8e5dcf6d4fcb7ff631", size = 235474, upload-time = "2026-02-23T10:42:47.721Z" } -wheels = [ - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a3/a1/10f04224542d6a57073c4f339b6763836a0899c98966f1d4ffcf56d2cf61/langfuse-3.14.5-py3-none-any.whl", hash = "sha256:5054b1c705ec69bce2d7077ce7419727ac629159428da013790979ca9cae77d5", size = 421240, upload-time = "2026-02-23T10:42:46.085Z" }, -] - [[package]] name = "langgraph" version = "1.0.6" @@ -3132,35 +3081,6 @@ wheels = [ { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e7/c3/26b8a0908a9db249de3b4169692e1c7c19048a9bc41a4d3209cee7dbb758/psycopg_pool-3.3.0-py3-none-any.whl", hash = "sha256:2e44329155c410b5e8666372db44276a8b1ebd8c90f1c3026ebba40d4bc81063", size = 39995, upload-time = "2025-12-01T11:34:29.761Z" }, ] -[[package]] -name = "pyarrow" -version = "23.0.1" -source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } -sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/88/22/134986a4cc224d593c1afde5494d18ff629393d74cc2eddb176669f234a4/pyarrow-23.0.1.tar.gz", hash = "sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019", size = 1167336, upload-time = "2026-02-16T10:14:12.39Z" } -wheels = [ - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9a/4b/4166bb5abbfe6f750fc60ad337c43ecf61340fa52ab386da6e8dbf9e63c4/pyarrow-23.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f4b0dbfa124c0bb161f8b5ebb40f1a680b70279aa0c9901d44a2b5a20806039f", size = 34214575, upload-time = "2026-02-16T10:09:56.225Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e1/da/3f941e3734ac8088ea588b53e860baeddac8323ea40ce22e3d0baa865cc9/pyarrow-23.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:7707d2b6673f7de054e2e83d59f9e805939038eebe1763fe811ee8fa5c0cd1a7", size = 35832540, upload-time = "2026-02-16T10:10:03.428Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/88/7c/3d841c366620e906d54430817531b877ba646310296df42ef697308c2705/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:86ff03fb9f1a320266e0de855dee4b17da6794c595d207f89bba40d16b5c78b9", size = 44470940, upload-time = "2026-02-16T10:10:10.704Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2c/a5/da83046273d990f256cb79796a190bbf7ec999269705ddc609403f8c6b06/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:813d99f31275919c383aab17f0f455a04f5a429c261cc411b1e9a8f5e4aaaa05", size = 47586063, upload-time = "2026-02-16T10:10:17.95Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/5b/3c/b7d2ebcff47a514f47f9da1e74b7949138c58cfeb108cdd4ee62f43f0cf3/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bf5842f960cddd2ef757d486041d57c96483efc295a8c4a0e20e704cbbf39c67", size = 48173045, upload-time = "2026-02-16T10:10:25.363Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/43/b2/b40961262213beaba6acfc88698eb773dfce32ecdf34d19291db94c2bd73/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564baf97c858ecc03ec01a41062e8f4698abc3e6e2acd79c01c2e97880a19730", size = 50621741, upload-time = "2026-02-16T10:10:33.477Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f6/70/1fdda42d65b28b078e93d75d371b2185a61da89dda4def8ba6ba41ebdeb4/pyarrow-23.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:07deae7783782ac7250989a7b2ecde9b3c343a643f82e8a4df03d93b633006f0", size = 27620678, upload-time = "2026-02-16T10:10:39.31Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/47/10/2cbe4c6f0fb83d2de37249567373d64327a5e4d8db72f486db42875b08f6/pyarrow-23.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6b8fda694640b00e8af3c824f99f789e836720aa8c9379fb435d4c4953a756b8", size = 34210066, upload-time = "2026-02-16T10:10:45.487Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/cb/4f/679fa7e84dadbaca7a65f7cdba8d6c83febbd93ca12fa4adf40ba3b6362b/pyarrow-23.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:8ff51b1addc469b9444b7c6f3548e19dc931b172ab234e995a60aea9f6e6025f", size = 35825526, upload-time = "2026-02-16T10:10:52.266Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f9/63/d2747d930882c9d661e9398eefc54f15696547b8983aaaf11d4a2e8b5426/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:71c5be5cbf1e1cb6169d2a0980850bccb558ddc9b747b6206435313c47c37677", size = 44473279, upload-time = "2026-02-16T10:11:01.557Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b3/93/10a48b5e238de6d562a411af6467e71e7aedbc9b87f8d3a35f1560ae30fb/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9b6f4f17b43bc39d56fec96e53fe89d94bac3eb134137964371b45352d40d0c2", size = 47585798, upload-time = "2026-02-16T10:11:09.401Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/5c/20/476943001c54ef078dbf9542280e22741219a184a0632862bca4feccd666/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fc13fc6c403d1337acab46a2c4346ca6c9dec5780c3c697cf8abfd5e19b6b37", size = 48179446, upload-time = "2026-02-16T10:11:17.781Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/4b/b6/5dd0c47b335fcd8edba9bfab78ad961bd0fd55ebe53468cc393f45e0be60/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c16ed4f53247fa3ffb12a14d236de4213a4415d127fe9cebed33d51671113e2", size = 50623972, upload-time = "2026-02-16T10:11:26.185Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d5/09/a532297c9591a727d67760e2e756b83905dd89adb365a7f6e9c72578bcc1/pyarrow-23.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:cecfb12ef629cf6be0b1887f9f86463b0dd3dc3195ae6224e74006be4736035a", size = 27540749, upload-time = "2026-02-16T10:12:23.297Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a5/8e/38749c4b1303e6ae76b3c80618f84861ae0c55dd3c2273842ea6f8258233/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:29f7f7419a0e30264ea261fdc0e5fe63ce5a6095003db2945d7cd78df391a7e1", size = 34471544, upload-time = "2026-02-16T10:11:32.535Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a3/73/f237b2bc8c669212f842bcfd842b04fc8d936bfc9d471630569132dc920d/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:33d648dc25b51fd8055c19e4261e813dfc4d2427f068bcecc8b53d01b81b0500", size = 35949911, upload-time = "2026-02-16T10:11:39.813Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0c/86/b912195eee0903b5611bf596833def7d146ab2d301afeb4b722c57ffc966/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd395abf8f91c673dd3589cadc8cc1ee4e8674fa61b2e923c8dd215d9c7d1f41", size = 44520337, upload-time = "2026-02-16T10:11:47.764Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/69/c2/f2a717fb824f62d0be952ea724b4f6f9372a17eed6f704b5c9526f12f2f1/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:00be9576d970c31defb5c32eb72ef585bf600ef6d0a82d5eccaae96639cf9d07", size = 47548944, upload-time = "2026-02-16T10:11:56.607Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/84/a7/90007d476b9f0dc308e3bc57b832d004f848fd6c0da601375d20d92d1519/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c2139549494445609f35a5cda4eb94e2c9e4d704ce60a095b342f82460c73a83", size = 48236269, upload-time = "2026-02-16T10:12:04.47Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b0/3f/b16fab3e77709856eb6ac328ce35f57a6d4a18462c7ca5186ef31b45e0e0/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7044b442f184d84e2351e5084600f0d7343d6117aabcbc1ac78eb1ae11eb4125", size = 50604794, upload-time = "2026-02-16T10:12:11.797Z" }, - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e9/a1/22df0620a9fac31d68397a75465c344e83c3dfe521f7612aea33e27ab6c0/pyarrow-23.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a35581e856a2fafa12f3f54fce4331862b1cfb0bef5758347a858a4aa9d6bae8", size = 27660642, upload-time = "2026-02-16T10:12:17.746Z" }, -] - [[package]] name = "pyasn1" version = "0.6.1" @@ -5161,7 +5081,6 @@ dependencies = [ { name = "langchain-openai" }, { name = "langchain-tavily" }, { name = "langchain-text-splitters" }, - { name = "langfuse" }, { name = "langgraph" }, { name = "langgraph-checkpoint-postgres" }, { name = "langgraph-checkpoint-sqlite" }, @@ -5212,10 +5131,6 @@ dependencies = [ dev = [ { name = "ruff" }, ] -eval = [ - { name = "datasets" }, - { name = "huggingface-hub" }, -] test = [ { name = "pytest" }, { name = "pytest-asyncio" }, @@ -5247,7 +5162,6 @@ requires-dist = [ { name = "langchain-openai", specifier = ">=1.0.2" }, { name = "langchain-tavily", specifier = ">=0.2.13" }, { name = "langchain-text-splitters", specifier = ">=1.0" }, - { name = "langfuse", specifier = ">=2.0.0" }, { name = "langgraph", specifier = ">=1.0.1" }, { name = "langgraph-checkpoint-postgres", specifier = ">=2.0.0" }, { name = "langgraph-checkpoint-sqlite", specifier = ">=3.0" }, @@ -5294,10 +5208,6 @@ requires-dist = [ [package.metadata.requires-dev] dev = [{ name = "ruff", specifier = ">=0.12.1" }] -eval = [ - { name = "datasets", specifier = ">=3.0.0" }, - { name = "huggingface-hub", specifier = ">=0.25.0" }, -] test = [ { name = "pytest", specifier = ">=8.0.0" }, { name = "pytest-asyncio", specifier = ">=0.24.0" },