feat(guard): 添加内容审查模块并集成至聊天服务（默认关闭）
- 新增内容审查关键词库 src/static/bad_keywords.txt，支持自定义敏感词 - 实现内容审查插件 src/plugins/guard.py，提供敏感词检测功能 - 配置新增 enable_content_guard 选项，支持开关内容审查功能 - 聊天路由 chat_router.py 集成输入和输出内容审查逻辑，拦截敏感内容 - 前端设置页增加内容审查开关，方便管理员启用或禁用该功能 - 错误时前端显示详细错误信息，并停止消息流，提升用户体验 - README 添加服务安全说明文档，介绍内容审查机制与配置方法
This commit is contained in:
Wenjie Zhang
parent
aebe63ce1c
commit
342c541645
@ -314,6 +314,14 @@ MCP_SERVERS = {
|
||||
}
|
||||
```
|
||||
|
||||
### 服务安全
|
||||
|
||||
为了保障服务内容的合规性，系统内置了一套内容审查机制。对用户的输入和模型生成的输出进行关键词过滤，防止不当内容的传播。
|
||||
|
||||
管理员可以在 `设置` -> `基本设置` 页面一键启用或禁用内容审查功能。**敏感词词库**位于 `src/static/bad_keywords.txt` 文件，可以根据需要自行修改，每行一个关键词。
|
||||
|
||||
|
||||
|
||||
### 服务端口说明
|
||||
|
||||
| 端口 | 服务 | 说明 |
|
||||
|
||||
@ -20,6 +20,7 @@ from src import config as conf
|
||||
from src.agents import agent_manager
|
||||
from src.agents.common.tools import gen_tool_info, get_buildin_tools
|
||||
from src.models import select_model
|
||||
from src.plugins.guard import content_guard
|
||||
from src.utils.logging_config import logger
|
||||
|
||||
chat = APIRouter(prefix="/chat", tags=["chat"])
|
||||
@ -142,6 +143,11 @@ async def chat_agent(
|
||||
# 代表服务端已经收到了请求
|
||||
yield make_chunk(status="init", meta=meta, msg=HumanMessage(content=query).model_dump())
|
||||
|
||||
# Input guard
|
||||
if conf.enable_content_guard and content_guard.check(query):
|
||||
yield make_chunk(status="error", message="输入内容包含敏感词", meta=meta)
|
||||
return
|
||||
|
||||
try:
|
||||
agent = agent_manager.get_agent(agent_id)
|
||||
except Exception as e:
|
||||
@ -158,9 +164,16 @@ async def chat_agent(
|
||||
input_context = {"user_id": user_id, "thread_id": thread_id}
|
||||
|
||||
try:
|
||||
# Output guard for streaming
|
||||
accumulated_content = ""
|
||||
async for msg, metadata in agent.stream_messages(messages, input_context=input_context):
|
||||
# logger.debug(f"msg: {msg.model_dump()}, metadata: {metadata}")
|
||||
if isinstance(msg, AIMessageChunk):
|
||||
accumulated_content += msg.content
|
||||
if conf.enable_content_guard and content_guard.check(accumulated_content):
|
||||
logger.warning(f"Sensitive content detected in stream: {accumulated_content}")
|
||||
yield make_chunk(message="检测到敏感内容，已中断输出", status="error")
|
||||
return
|
||||
yield make_chunk(content=msg.content, msg=msg.model_dump(), metadata=metadata, status="loading")
|
||||
else:
|
||||
yield make_chunk(msg=msg.model_dump(), metadata=metadata, status="loading")
|
||||
|
||||
@ -49,6 +49,7 @@ class Config(SimpleConfig):
|
||||
### >>> 默认配置
|
||||
# 功能选项
|
||||
self.add_item("enable_reranker", default=False, des="是否开启重排序")
|
||||
self.add_item("enable_content_guard", default=False, des="是否启用内容审查")
|
||||
self.add_item(
|
||||
"enable_web_search",
|
||||
default=False,
|
||||
|
||||
34
src/plugins/guard.py
Normal file
34
src/plugins/guard.py
Normal file
@ -0,0 +1,34 @@
|
||||
import os
|
||||
from typing import List
|
||||
|
||||
def load_keywords(file_path: str) -> List[str]:
|
||||
"""Loads keywords from a file, one per line."""
|
||||
if not os.path.exists(file_path):
|
||||
keywords = []
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
keywords = [line.strip() for line in f if line.strip() and not line.startswith("#")]
|
||||
|
||||
return keywords
|
||||
|
||||
class ContentGuard:
|
||||
def __init__(self, keywords_file: str = "src/static/bad_keywords.txt"):
|
||||
self.keywords = load_keywords(keywords_file)
|
||||
if not self.keywords:
|
||||
# Default keywords if the file is empty or not found
|
||||
self.keywords = ["贩毒"]
|
||||
|
||||
def check(self, text: str) -> bool:
|
||||
"""
|
||||
Checks if the text contains any sensitive keywords.
|
||||
Returns True if sensitive content is found, False otherwise.
|
||||
"""
|
||||
if not text:
|
||||
return False
|
||||
text_lower = text.lower()
|
||||
for keyword in self.keywords:
|
||||
if keyword in text_lower:
|
||||
return True
|
||||
return False
|
||||
|
||||
# Global instance
|
||||
content_guard = ContentGuard()
|
||||
129
src/static/bad_keywords.txt
Normal file
129
src/static/bad_keywords.txt
Normal file
@ -0,0 +1,129 @@
|
||||
# ==========================================================
|
||||
# 有害内容过滤关键词列表
|
||||
# 说明:
|
||||
# 1. 本列表仅为基础示例，请根据您的具体应用场景进行修改和扩展。
|
||||
# 2. 很多词语具有多义性，请务必结合语境进行判断，以防误伤。
|
||||
# 3. 网络用语和暗语变化迅速，建议定期更新此列表。
|
||||
# ==========================================================
|
||||
|
||||
# ----------------------------------------
|
||||
# 1. 仇恨言论 (Hate Speech)
|
||||
# ----------------------------------------
|
||||
|
||||
# ** 种族与民族歧视 **
|
||||
尼哥
|
||||
黑鬼
|
||||
白皮猪
|
||||
棒子
|
||||
阿三
|
||||
|
||||
# ** 宗教歧视 **
|
||||
异教徒
|
||||
邪教
|
||||
|
||||
# ** 性别与性取向歧视 **
|
||||
死娘炮
|
||||
男人婆
|
||||
同性恋不得好死
|
||||
恐同
|
||||
|
||||
|
||||
# ----------------------------------------
|
||||
# 2. 暴力与极端内容 (Violence & Extremism)
|
||||
# ----------------------------------------
|
||||
|
||||
# ** 煽动暴力 **
|
||||
杀全家
|
||||
弄死他
|
||||
打断腿
|
||||
血洗
|
||||
人肉他
|
||||
|
||||
# ** 恐怖主义与极端思想 **
|
||||
圣战
|
||||
ISIS
|
||||
东突
|
||||
|
||||
# ** 血腥与残忍 **
|
||||
虐杀
|
||||
活剐
|
||||
分尸
|
||||
电锯惊魂
|
||||
|
||||
# ** 武器与爆炸物 **
|
||||
土制炸弹
|
||||
买枪
|
||||
军火
|
||||
C4炸药
|
||||
炸药爆
|
||||
制作炸药
|
||||
制作武器
|
||||
炸弹
|
||||
|
||||
# ----------------------------------------
|
||||
# 3. 非法活动与违禁品 (Illegal Activities)
|
||||
# ----------------------------------------
|
||||
|
||||
# ** 毒品 **
|
||||
冰毒
|
||||
K粉
|
||||
海洛因
|
||||
吸毒
|
||||
贩毒
|
||||
摇头丸
|
||||
毒品走私
|
||||
|
||||
|
||||
# ** 赌博 **
|
||||
线上赌场
|
||||
澳门首家
|
||||
性感荷官
|
||||
百家乐
|
||||
六合彩
|
||||
|
||||
# ** 诈骗 **
|
||||
杀猪盘
|
||||
刷单兼职
|
||||
网贷陷阱
|
||||
冒充公检法
|
||||
|
||||
# ** 违禁品交易 **
|
||||
出售个人信息
|
||||
办假证
|
||||
针孔摄像头
|
||||
窃听器
|
||||
|
||||
|
||||
# ----------------------------------------
|
||||
# 4. 色情与成人内容 (Pornography)
|
||||
# ----------------------------------------
|
||||
AV女优
|
||||
草榴
|
||||
91大神
|
||||
黄片
|
||||
色情网站
|
||||
约炮
|
||||
裸聊
|
||||
福利姬
|
||||
|
||||
# ----------------------------------------
|
||||
# 5. 自残与危险行为 (Self-harm)
|
||||
# ----------------------------------------
|
||||
自杀教程
|
||||
割腕
|
||||
烧炭
|
||||
无痛自杀
|
||||
一起死
|
||||
|
||||
# ----------------------------------------
|
||||
# 6. 网络欺凌与骚扰 (Cyberbullying)
|
||||
# ----------------------------------------
|
||||
biss
|
||||
nmsl
|
||||
开盒
|
||||
挂人
|
||||
孤儿
|
||||
|
||||
# ==========================================================
|
||||
# 列表结束
|
||||
# ==========================================================
|
||||
@ -34,7 +34,6 @@ body {
|
||||
.layout-container {
|
||||
width: 100%;
|
||||
padding: 0px 30px;
|
||||
background-color: #FAFCFD;
|
||||
|
||||
h2 {
|
||||
margin: 20px 0 10px 0;
|
||||
|
||||
@ -381,7 +381,31 @@ const _processStreamChunk = (chunk, threadId) => {
|
||||
break;
|
||||
case 'error':
|
||||
handleChatError({ message }, 'stream');
|
||||
resetOnGoingConv(threadId);
|
||||
// Stop the loading indicator
|
||||
if (threadState) {
|
||||
threadState.isStreaming = false;
|
||||
|
||||
// Create a new AI message chunk for the error
|
||||
const errorMsgChunk = {
|
||||
id: 'ai-error-' + Date.now(),
|
||||
type: 'ai',
|
||||
role: 'assistant',
|
||||
content: chunk.message || 'An error occurred',
|
||||
isError: true // Custom flag for styling
|
||||
};
|
||||
|
||||
// Add this to the chunks of the ongoing conversation
|
||||
if (threadState.onGoingConv && threadState.onGoingConv.msgChunks) {
|
||||
threadState.onGoingConv.msgChunks[errorMsgChunk.id] = [errorMsgChunk];
|
||||
}
|
||||
|
||||
// Abort the stream controller to stop processing further events
|
||||
if (threadState.streamAbortController) {
|
||||
threadState.streamAbortController.abort();
|
||||
threadState.streamAbortController = null;
|
||||
}
|
||||
}
|
||||
// We no longer call resetOnGoingConv to keep the context.
|
||||
break;
|
||||
case 'finished':
|
||||
fetchThreadMessages({ agentId: currentAgentId.value, threadId: threadId });
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
<template>
|
||||
<div class="">
|
||||
<div class="setting-view">
|
||||
<HeaderComponent title="设置" class="setting-header">
|
||||
|
||||
<template #actions>
|
||||
@ -49,6 +49,13 @@
|
||||
</a-select-option>
|
||||
</a-select>
|
||||
</div>
|
||||
<div class="card">
|
||||
<span class="label">{{ items?.enable_content_guard.des }}</span>
|
||||
<a-switch
|
||||
:checked="configStore.config?.enable_content_guard"
|
||||
@change="handleChange('enable_content_guard', $event)"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 服务链接部分 -->
|
||||
@ -225,8 +232,10 @@ const openLink = (url) => {
|
||||
</script>
|
||||
|
||||
<style lang="less" scoped>
|
||||
|
||||
.setting-container {
|
||||
--setting-header-height: 65px;
|
||||
--setting-header-height: 55px;
|
||||
max-width: 870px;
|
||||
}
|
||||
|
||||
.setting-header {
|
||||
@ -246,7 +255,7 @@ const openLink = (url) => {
|
||||
}
|
||||
|
||||
.sider {
|
||||
width: 200px;
|
||||
width: 180px;
|
||||
height: 100%;
|
||||
padding: 0 20px;
|
||||
position: sticky;
|
||||
@ -254,7 +263,6 @@ const openLink = (url) => {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
border-right: 1px solid var(--main-20);
|
||||
gap: 8px;
|
||||
padding-top: 20px;
|
||||
|
||||
@ -271,12 +279,12 @@ const openLink = (url) => {
|
||||
color: var(--gray-700);
|
||||
|
||||
&:hover {
|
||||
background: var(--gray-100);
|
||||
background: var(--gray-50);
|
||||
}
|
||||
|
||||
&.activesec {
|
||||
background: var(--gray-200);
|
||||
color: var(--gray-900);
|
||||
background: var(--gray-100);
|
||||
color: var(--main-700);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user