98 lines
2.6 KiB
Python
98 lines
2.6 KiB
Python
"""GAIA 答案评分模块
|
||
|
||
实现 GAIA 官方的 quasi exact match 评分逻辑。
|
||
参考: https://huggingface.co/spaces/gaia-benchmark/leaderboard
|
||
"""
|
||
|
||
import re
|
||
import string
|
||
|
||
|
||
class GaiaScorer:
|
||
"""GAIA 准精确匹配评分器"""
|
||
|
||
@staticmethod
|
||
def normalize_text(text: str) -> str:
|
||
"""文本标准化
|
||
|
||
- 去除首尾空白
|
||
- 统一为小写
|
||
- 移除冠词 (a, an, the)
|
||
- 移除标点
|
||
- 合并连续空白
|
||
"""
|
||
text = text.strip().lower()
|
||
|
||
# 移除冠词
|
||
text = re.sub(r"\b(a|an|the)\b", " ", text)
|
||
|
||
# 移除标点
|
||
text = text.translate(str.maketrans("", "", string.punctuation))
|
||
|
||
# 合并连续空白
|
||
text = re.sub(r"\s+", " ", text).strip()
|
||
|
||
return text
|
||
|
||
@staticmethod
|
||
def normalize_number(text: str) -> str:
|
||
"""数字标准化
|
||
|
||
- "1,000" → "1000"
|
||
- "3.0" → "3"
|
||
- "$100" → "100"
|
||
- "100%" → "100"
|
||
"""
|
||
# 去除货币符号
|
||
text = re.sub(r"[$€£¥]", "", text)
|
||
# 去除百分号
|
||
text = text.rstrip("%")
|
||
# 去除千分位逗号
|
||
text = text.replace(",", "")
|
||
|
||
# 判断是否为数字，若是则标准化
|
||
try:
|
||
num = float(text)
|
||
# 如果是整数则去掉 .0
|
||
if num == int(num):
|
||
return str(int(num))
|
||
return str(num)
|
||
except ValueError:
|
||
return text
|
||
|
||
@classmethod
|
||
def score(cls, prediction: str, gold: str) -> bool:
|
||
"""计算单条评估的准精确匹配分数
|
||
|
||
Args:
|
||
prediction: 模型预测答案
|
||
gold: 黄金标准答案
|
||
|
||
Returns:
|
||
True 表示匹配，False 表示不匹配
|
||
"""
|
||
if not prediction or not gold:
|
||
return False
|
||
|
||
# 先尝试数字比较
|
||
pred_num = cls.normalize_number(prediction.strip())
|
||
gold_num = cls.normalize_number(gold.strip())
|
||
try:
|
||
if float(pred_num) == float(gold_num):
|
||
return True
|
||
except ValueError:
|
||
pass
|
||
|
||
# 检查是否为列表答案（逗号分隔）
|
||
if "," in gold:
|
||
pred_items = sorted(cls.normalize_text(item) for item in prediction.split(","))
|
||
gold_items = sorted(cls.normalize_text(item) for item in gold.split(","))
|
||
if pred_items == gold_items:
|
||
return True
|
||
|
||
# 文本标准化比较
|
||
pred_normalized = cls.normalize_text(prediction)
|
||
gold_normalized = cls.normalize_text(gold)
|
||
|
||
return pred_normalized == gold_normalized
|