diff --git a/README.md b/README.md index 2cc8ad38..4a7991b2 100644 --- a/README.md +++ b/README.md @@ -314,6 +314,14 @@ MCP_SERVERS = { } ``` +### 服务安全 + +为了保障服务内容的合规性，系统内置了一套内容审查机制。对用户的输入和模型生成的输出进行关键词过滤，防止不当内容的传播。 + +管理员可以在 `设置` -> `基本设置` 页面一键启用或禁用内容审查功能。**敏感词词库**位于 `src/static/bad_keywords.txt` 文件，可以根据需要自行修改，每行一个关键词。 + + + ### 服务端口说明 | 端口 | 服务 | 说明 | diff --git a/server/routers/chat_router.py b/server/routers/chat_router.py index ae5a9c2c..4431a717 100644 --- a/server/routers/chat_router.py +++ b/server/routers/chat_router.py @@ -20,6 +20,7 @@ from src import config as conf from src.agents import agent_manager from src.agents.common.tools import gen_tool_info, get_buildin_tools from src.models import select_model +from src.plugins.guard import content_guard from src.utils.logging_config import logger chat = APIRouter(prefix="/chat", tags=["chat"]) @@ -142,6 +143,11 @@ async def chat_agent( # 代表服务端已经收到了请求 yield make_chunk(status="init", meta=meta, msg=HumanMessage(content=query).model_dump()) + # Input guard + if conf.enable_content_guard and content_guard.check(query): + yield make_chunk(status="error", message="输入内容包含敏感词", meta=meta) + return + try: agent = agent_manager.get_agent(agent_id) except Exception as e: @@ -158,9 +164,16 @@ async def chat_agent( input_context = {"user_id": user_id, "thread_id": thread_id} try: + # Output guard for streaming + accumulated_content = "" async for msg, metadata in agent.stream_messages(messages, input_context=input_context): # logger.debug(f"msg: {msg.model_dump()}, metadata: {metadata}") if isinstance(msg, AIMessageChunk): + accumulated_content += msg.content + if conf.enable_content_guard and content_guard.check(accumulated_content): + logger.warning(f"Sensitive content detected in stream: {accumulated_content}") + yield make_chunk(message="检测到敏感内容，已中断输出", status="error") + return yield make_chunk(content=msg.content, msg=msg.model_dump(), metadata=metadata, status="loading") else: yield make_chunk(msg=msg.model_dump(), metadata=metadata, status="loading") diff --git a/src/config/app.py b/src/config/app.py index 3d601948..1fd41489 100644 --- a/src/config/app.py +++ b/src/config/app.py @@ -49,6 +49,7 @@ class Config(SimpleConfig): ### >>> 默认配置 # 功能选项 self.add_item("enable_reranker", default=False, des="是否开启重排序") + self.add_item("enable_content_guard", default=False, des="是否启用内容审查") self.add_item( "enable_web_search", default=False, diff --git a/src/plugins/guard.py b/src/plugins/guard.py new file mode 100644 index 00000000..3aa66b34 --- /dev/null +++ b/src/plugins/guard.py @@ -0,0 +1,34 @@ +import os +from typing import List + +def load_keywords(file_path: str) -> List[str]: + """Loads keywords from a file, one per line.""" + if not os.path.exists(file_path): + keywords = [] + with open(file_path, "r", encoding="utf-8") as f: + keywords = [line.strip() for line in f if line.strip() and not line.startswith("#")] + + return keywords + +class ContentGuard: + def __init__(self, keywords_file: str = "src/static/bad_keywords.txt"): + self.keywords = load_keywords(keywords_file) + if not self.keywords: + # Default keywords if the file is empty or not found + self.keywords = ["贩毒"] + + def check(self, text: str) -> bool: + """ + Checks if the text contains any sensitive keywords. + Returns True if sensitive content is found, False otherwise. + """ + if not text: + return False + text_lower = text.lower() + for keyword in self.keywords: + if keyword in text_lower: + return True + return False + +# Global instance +content_guard = ContentGuard() diff --git a/src/static/bad_keywords.txt b/src/static/bad_keywords.txt new file mode 100644 index 00000000..58c7f358 --- /dev/null +++ b/src/static/bad_keywords.txt @@ -0,0 +1,129 @@ +# ========================================================== +# 有害内容过滤关键词列表 +# 说明: +# 1. 本列表仅为基础示例，请根据您的具体应用场景进行修改和扩展。 +# 2. 很多词语具有多义性，请务必结合语境进行判断，以防误伤。 +# 3. 网络用语和暗语变化迅速，建议定期更新此列表。 +# ========================================================== + +# ---------------------------------------- +# 1. 仇恨言论 (Hate Speech) +# ---------------------------------------- + +# ** 种族与民族歧视 ** +尼哥 +黑鬼 +白皮猪 +棒子 +阿三 + +# ** 宗教歧视 ** +异教徒 +邪教 + +# ** 性别与性取向歧视 ** +死娘炮 +男人婆 +同性恋不得好死 +恐同 + + +# ---------------------------------------- +# 2. 暴力与极端内容 (Violence & Extremism) +# ---------------------------------------- + +# ** 煽动暴力 ** +杀全家 +弄死他 +打断腿 +血洗 +人肉他 + +# ** 恐怖主义与极端思想 ** +圣战 +ISIS +东突 + +# ** 血腥与残忍 ** +虐杀 +活剐 +分尸 +电锯惊魂 + +# ** 武器与爆炸物 ** +土制炸弹 +买枪 +军火 +C4炸药 +炸药爆 +制作炸药 +制作武器 +炸弹 + +# ---------------------------------------- +# 3. 非法活动与违禁品 (Illegal Activities) +# ---------------------------------------- + +# ** 毒品 ** +冰毒 +K粉 +海洛因 +吸毒 +贩毒 +摇头丸 +毒品走私 + + +# ** 赌博 ** +线上赌场 +澳门首家 +性感荷官 +百家乐 +六合彩 + +# ** 诈骗 ** +杀猪盘 +刷单兼职 +网贷陷阱 +冒充公检法 + +# ** 违禁品交易 ** +出售个人信息 +办假证 +针孔摄像头 +窃听器 + + +# ---------------------------------------- +# 4. 色情与成人内容 (Pornography) +# ---------------------------------------- +AV女优 +草榴 +91大神 +黄片 +色情网站 +约炮 +裸聊 +福利姬 + +# ---------------------------------------- +# 5. 自残与危险行为 (Self-harm) +# ---------------------------------------- +自杀教程 +割腕 +烧炭 +无痛自杀 +一起死 + +# ---------------------------------------- +# 6. 网络欺凌与骚扰 (Cyberbullying) +# ---------------------------------------- +biss +nmsl +开盒 +挂人 +孤儿 + +# ========================================================== +# 列表结束 +# ========================================================== \ No newline at end of file diff --git a/web/src/assets/css/main.css b/web/src/assets/css/main.css index 41b11c9f..74ebf6db 100644 --- a/web/src/assets/css/main.css +++ b/web/src/assets/css/main.css @@ -34,7 +34,6 @@ body { .layout-container { width: 100%; padding: 0px 30px; - background-color: #FAFCFD; h2 { margin: 20px 0 10px 0; diff --git a/web/src/components/AgentChatComponent.vue b/web/src/components/AgentChatComponent.vue index 2758703e..5ae130fc 100644 --- a/web/src/components/AgentChatComponent.vue +++ b/web/src/components/AgentChatComponent.vue @@ -381,7 +381,31 @@ const _processStreamChunk = (chunk, threadId) => { break; case 'error': handleChatError({ message }, 'stream'); - resetOnGoingConv(threadId); + // Stop the loading indicator + if (threadState) { + threadState.isStreaming = false; + + // Create a new AI message chunk for the error + const errorMsgChunk = { + id: 'ai-error-' + Date.now(), + type: 'ai', + role: 'assistant', + content: chunk.message || 'An error occurred', + isError: true // Custom flag for styling + }; + + // Add this to the chunks of the ongoing conversation + if (threadState.onGoingConv && threadState.onGoingConv.msgChunks) { + threadState.onGoingConv.msgChunks[errorMsgChunk.id] = [errorMsgChunk]; + } + + // Abort the stream controller to stop processing further events + if (threadState.streamAbortController) { + threadState.streamAbortController.abort(); + threadState.streamAbortController = null; + } + } + // We no longer call resetOnGoingConv to keep the context. break; case 'finished': fetchThreadMessages({ agentId: currentAgentId.value, threadId: threadId }); diff --git a/web/src/views/SettingView.vue b/web/src/views/SettingView.vue index 24da3450..0b087447 100644 --- a/web/src/views/SettingView.vue +++ b/web/src/views/SettingView.vue @@ -1,5 +1,5 @@