58 lines
1.9 KiB
Python
58 lines
1.9 KiB
Python
|
"""GAIA 评估配置管理模块"""
|
|||
|
|
|
|||
|
|
from dataclasses import dataclass, field
|
|||
|
|
from pathlib import Path
|
|||
|
|
|
|||
|
|
|
|||
|
|
@dataclass
|
|||
|
|
class EvalConfig:
|
|||
|
|
"""GAIA 评估配置
|
|||
|
|
|
|||
|
|
配置优先级：CLI 参数 > 环境变量 > 默认值
|
|||
|
|
当指定 agent_config_id 时，会从数据库加载预设配置（model/tools/knowledges/system_prompt 等）
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
# Agent 相关
|
|||
|
|
agent_id: str = "ChatbotAgent"
|
|||
|
|
agent_config_id: int | None = None # 从数据库加载预设配置
|
|||
|
|
|
|||
|
|
# 备选配置（当未指定 agent_config_id 时使用）
|
|||
|
|
model: str | None = None
|
|||
|
|
system_prompt: str | None = None
|
|||
|
|
tools: list[dict] | None = None
|
|||
|
|
knowledges: list[str] | None = None
|
|||
|
|
|
|||
|
|
# 数据集相关
|
|||
|
|
level: str = "all" # "1", "2", "3", "all"
|
|||
|
|
split: str = "validation" # "validation" or "test"
|
|||
|
|
max_samples: int | None = None # 限制评估样本数（调试用）
|
|||
|
|
|
|||
|
|
# 运行控制
|
|||
|
|
timeout: int = 300 # 单题超时，秒
|
|||
|
|
concurrency: int = 1 # 并发评估数
|
|||
|
|
|
|||
|
|
# 输出相关
|
|||
|
|
output_dir: Path = field(default_factory=lambda: Path("eval_results"))
|
|||
|
|
|
|||
|
|
def get_dataset_configs(self) -> list[str]:
|
|||
|
|
"""根据 level 获取 HuggingFace dataset config 名称列表"""
|
|||
|
|
if self.level == "all":
|
|||
|
|
return ["2023_level1", "2023_level2", "2023_level3"]
|
|||
|
|
return [f"2023_level{self.level}"]
|
|||
|
|
|
|||
|
|
def build_agent_config(self) -> dict:
|
|||
|
|
"""构建不使用 agent_config_id 时的 agent_config 字典
|
|||
|
|
|
|||
|
|
当未指定 agent_config_id 时，从 CLI 参数组装配置。
|
|||
|
|
"""
|
|||
|
|
config = {}
|
|||
|
|
if self.model:
|
|||
|
|
config["model"] = self.model
|
|||
|
|
if self.system_prompt:
|
|||
|
|
config["system_prompt"] = self.system_prompt
|
|||
|
|
if self.tools is not None:
|
|||
|
|
config["tools"] = self.tools
|
|||
|
|
if self.knowledges is not None:
|
|||
|
|
config["knowledges"] = self.knowledges
|
|||
|
|
return config
|