From 253cc0b149b435d236e862fe445375524e4a7457 Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Fri, 12 Jun 2026 00:03:04 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=A4=9A=E6=A8=A1?= =?UTF-8?q?=E6=80=81=E5=9B=BE=E7=89=87=E4=B8=8A=E4=BC=A0=E5=92=8C=E5=8E=86?= =?UTF-8?q?=E5=8F=B2=E6=B8=B2=E6=9F=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../yuxi/services/agent_run_service.py | 2 +- backend/package/yuxi/services/file_preview.py | 26 ++++ backend/package/yuxi/utils/image_processor.py | 28 +++-- backend/server/routers/chat_router.py | 4 +- .../test/integration/api/test_chat_router.py | 55 +++++++++ .../unit/services/test_agent_run_service.py | 39 ++++++ .../test/unit/utils/test_image_processor.py | 31 +++++ docs/develop-guides/changelog.md | 1 + web/src/components/AgentChatComponent.vue | 111 ++++++++++-------- web/src/components/AgentInputArea.vue | 49 +------- web/src/components/AgentMessageComponent.vue | 40 +------ web/src/components/ImagePreviewComponent.vue | 38 ++++-- web/src/composables/useAgentStreamHandler.js | 24 ++-- web/src/utils/file_utils.js | 13 +- 14 files changed, 293 insertions(+), 168 deletions(-) create mode 100644 backend/test/unit/utils/test_image_processor.py diff --git a/backend/package/yuxi/services/agent_run_service.py b/backend/package/yuxi/services/agent_run_service.py index e1723b7b..770fe8eb 100644 --- a/backend/package/yuxi/services/agent_run_service.py +++ b/backend/package/yuxi/services/agent_run_service.py @@ -351,7 +351,7 @@ async def create_agent_run_view( conversation_id=conversation.id, role="user", content=input_content, - message_type="resume" if run_type == "resume" else "text", + message_type="resume" if run_type == "resume" else "multimodal_image" if image_content else "text", image_content=image_content, run_id=run_id, request_id=request_id, diff --git a/backend/package/yuxi/services/file_preview.py b/backend/package/yuxi/services/file_preview.py index 4bbb82e3..2b24e184 100644 --- a/backend/package/yuxi/services/file_preview.py +++ b/backend/package/yuxi/services/file_preview.py @@ -100,3 +100,29 @@ def detect_preview_type(path: str, raw_content: bytes) -> tuple[str, bool, str | return "text", True, None except UnicodeDecodeError: return "unsupported", False, "当前文件不是可读文本，暂不支持预览" + + +def detect_media_type(path: str, raw_content: bytes | None = None) -> str: + """Detect response media type, preferring file signatures over filename suffixes.""" + head = (raw_content or b"")[:512] + + if head.startswith(b"\x89PNG\r\n\x1a\n"): + return "image/png" + if head.startswith(b"\xff\xd8\xff"): + return "image/jpeg" + if head.startswith(b"GIF87a") or head.startswith(b"GIF89a"): + return "image/gif" + if head.startswith(b"RIFF") and b"WEBP" in head[:16]: + return "image/webp" + if head.startswith(b"BM"): + return "image/bmp" + if head.startswith(b"%PDF-"): + return "application/pdf" + + stripped_head = head.lstrip() + if stripped_head.startswith(b" bytes: """生成缩略图""" try: - # 创建副本以避免修改原图 - thumbnail = img.copy() - - # 转换为RGB模式（处理RGBA等格式） - if thumbnail.mode != "RGB": - thumbnail = thumbnail.convert("RGB") + thumbnail = self._convert_to_rgb_for_export(img) # 生成缩略图，保持宽高比 thumbnail.thumbnail(self.THUMBNAIL_SIZE, Image.Resampling.LANCZOS) @@ -129,6 +124,20 @@ class ImageProcessor: empty_img.save(output, format="JPEG", quality=85) return output.getvalue() + def _convert_to_rgb_for_export(self, img: Image.Image) -> Image.Image: + """转换为 RGB，同时把透明像素按白底合成，避免隐藏颜色变成可见像素。""" + if img.mode == "RGB": + return img.copy() + + has_alpha = img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info) + if not has_alpha: + return img.convert("RGB") + + rgba_img = img.convert("RGBA") + background = Image.new("RGBA", rgba_img.size, (255, 255, 255, 255)) + background.alpha_composite(rgba_img) + return background.convert("RGB") + def _compress_image(self, img: Image.Image, original_format: str) -> tuple[bytes, str]: """ 压缩图片，如果超过大小限制 @@ -140,12 +149,7 @@ class ImageProcessor: Returns: Tuple[bytes, str]: (压缩后的图片数据, 最终格式) """ - # 创建副本 - processed_img = img.copy() - - # 转换为RGB模式（如果需要） - if processed_img.mode in ("RGBA", "LA", "P"): - processed_img = processed_img.convert("RGB") + processed_img = self._convert_to_rgb_for_export(img) # 尝试保持原始格式，但优先使用JPEG（更好的压缩） target_format = "JPEG" if original_format != "PNG" else "PNG" diff --git a/backend/server/routers/chat_router.py b/backend/server/routers/chat_router.py index 557a61d9..aab412aa 100644 --- a/backend/server/routers/chat_router.py +++ b/backend/server/routers/chat_router.py @@ -1,7 +1,6 @@ import traceback import uuid from typing import Any -from mimetypes import guess_type from fastapi import APIRouter, Body, Depends, HTTPException, Query, UploadFile, File from fastapi.responses import FileResponse, StreamingResponse @@ -27,6 +26,7 @@ from yuxi.services.conversation_service import ( upload_thread_attachment_view, upload_tmp_attachment_view, ) +from yuxi.services.file_preview import detect_media_type from yuxi.services.thread_files_service import ( list_thread_files_view, read_thread_file_content_view, @@ -544,7 +544,7 @@ async def get_thread_artifact( path=path, ) - media_type = guess_type(file_path.name)[0] or "application/octet-stream" + media_type = detect_media_type(file_path.name, file_path.read_bytes()) headers = {"Content-Disposition": f'attachment; filename="{file_path.name}"'} if download else None return FileResponse(path=file_path, media_type=media_type, headers=headers) diff --git a/backend/test/integration/api/test_chat_router.py b/backend/test/integration/api/test_chat_router.py index b73c10b6..e0ce89e6 100644 --- a/backend/test/integration/api/test_chat_router.py +++ b/backend/test/integration/api/test_chat_router.py @@ -4,9 +4,12 @@ Integration tests for chat router endpoints. from __future__ import annotations +import base64 +import io import uuid import pytest +from PIL import Image from yuxi.agents.backends.sandbox import ensure_thread_dirs, sandbox_user_data_dir, sandbox_workspace_dir pytestmark = [pytest.mark.asyncio, pytest.mark.integration] @@ -17,6 +20,58 @@ async def test_chat_endpoints_require_authentication(test_client): assert (await test_client.get("/api/agent")).status_code == 401 +async def test_image_upload_composites_transparent_png_pixels_on_white(test_client, admin_headers): + image = Image.new("RGBA", (2, 2), (255, 255, 255, 0)) + image.putpixel((0, 0), (50, 87, 244, 0)) + image.putpixel((1, 0), (50, 87, 244, 255)) + + with io.BytesIO() as buffer: + image.save(buffer, format="PNG") + image_bytes = buffer.getvalue() + + response = await test_client.post( + "/api/chat/image/upload", + headers=admin_headers, + files={"file": ("transparent.png", image_bytes, "image/png")}, + ) + + assert response.status_code == 200, response.text + payload = response.json() + assert payload["success"] is True + assert payload["mime_type"] == "image/png" + + processed_data = base64.b64decode(payload["image_content"]) + with Image.open(io.BytesIO(processed_data)) as processed_image: + rgb_image = processed_image.convert("RGB") + + assert rgb_image.getpixel((0, 0)) == (255, 255, 255) + assert rgb_image.getpixel((1, 0)) == (50, 87, 244) + + +async def test_thread_artifact_uses_image_signature_for_content_type(test_client, admin_headers): + thread_id = await _create_thread_for_user(test_client, admin_headers) + image = Image.new("RGBA", (2, 2), (255, 255, 255, 0)) + + with io.BytesIO() as buffer: + image.save(buffer, format="PNG") + image_bytes = buffer.getvalue() + + upload_response = await test_client.post( + f"/api/chat/thread/{thread_id}/attachments", + headers=admin_headers, + files={"file": ("mislabeled.jpg", image_bytes, "image/jpeg")}, + ) + + assert upload_response.status_code == 200, upload_response.text + attachment = upload_response.json() + + artifact_response = await test_client.get(attachment["original_artifact_url"], headers=admin_headers) + + assert artifact_response.status_code == 200, artifact_response.text + assert artifact_response.headers["content-type"].startswith("image/png") + assert artifact_response.content.startswith(b"\x89PNG\r\n\x1a\n") + + async def _create_thread_for_user(test_client, headers: dict[str, str]) -> str: agents_resp = await test_client.get("/api/agent", headers=headers) assert agents_resp.status_code == 200, agents_resp.text diff --git a/backend/test/unit/services/test_agent_run_service.py b/backend/test/unit/services/test_agent_run_service.py index d0ef2647..6b140839 100644 --- a/backend/test/unit/services/test_agent_run_service.py +++ b/backend/test/unit/services/test_agent_run_service.py @@ -766,6 +766,45 @@ async def test_create_chat_run_persists_validated_model_spec(monkeypatch: pytest assert captured["input_payload"]["model_spec"] == "claude-x" +@pytest.mark.asyncio +async def test_create_chat_run_with_image_persists_multimodal_message_type(monkeypatch: pytest.MonkeyPatch): + captured = {} + created_run = SimpleNamespace(id="", thread_id="thread-1", status="pending", request_id="req-1", uid="user-1") + + class RunRepo: + def __init__(self, db_session): + del db_session + + async def get_run_by_request_id(self, request_id: str): + del request_id + return None + + async def create_run(self, **kwargs): + captured["input_payload"] = kwargs["input_payload"] + created_run.id = kwargs["run_id"] + return created_run + + async def set_input_message(self, run_id: str, message_id: int): + del run_id, message_id + return created_run + + db = _patch_common_run_repos(monkeypatch, RunRepo) + + await agent_run_service.create_agent_run_view( + query="看图", + agent_id="default", + thread_id="thread-1", + meta={"request_id": "req-1"}, + image_content="base64-image", + current_uid="user-1", + db=db, + ) + + assert captured["input_payload"]["image_content"] == "base64-image" + assert db.added[0].message_type == "multimodal_image" + assert db.added[0].image_content == "base64-image" + + @pytest.mark.asyncio async def test_create_chat_run_snapshots_agent_configured_model_spec(monkeypatch: pytest.MonkeyPatch): captured = {} diff --git a/backend/test/unit/utils/test_image_processor.py b/backend/test/unit/utils/test_image_processor.py new file mode 100644 index 00000000..1ab1b06a --- /dev/null +++ b/backend/test/unit/utils/test_image_processor.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import base64 +import io + +from PIL import Image + +from yuxi.utils.image_processor import process_uploaded_image + + +def test_process_uploaded_image_composites_transparent_png_pixels_on_white(): + image = Image.new("RGBA", (2, 2), (255, 255, 255, 0)) + image.putpixel((0, 0), (50, 87, 244, 0)) + image.putpixel((1, 0), (50, 87, 244, 255)) + + with io.BytesIO() as buffer: + image.save(buffer, format="PNG") + image_data = buffer.getvalue() + + result = process_uploaded_image(image_data, "transparent.png") + + assert result["success"] is True + assert result["format"] == "PNG" + assert result["mime_type"] == "image/png" + + processed_data = base64.b64decode(result["image_content"]) + with Image.open(io.BytesIO(processed_data)) as processed_image: + rgb_image = processed_image.convert("RGB") + + assert rgb_image.getpixel((0, 0)) == (255, 255, 255) + assert rgb_image.getpixel((1, 0)) == (50, 87, 244) diff --git a/docs/develop-guides/changelog.md b/docs/develop-guides/changelog.md index 560d5da5..0bb7e38b 100644 --- a/docs/develop-guides/changelog.md +++ b/docs/develop-guides/changelog.md @@ -59,6 +59,7 @@ - 工作区知识库分类显示：知识库侧边栏按创建者分组为“我的知识库”和“共享知识库”，自己创建的知识库显示在“我的知识库”下，非自己创建的显示在“共享知识库”下；`knowledge_bases` 表新增 `created_by` 字段记录创建者 uid。 - 工作区文件上传支持多选：`/workspace/upload` 与 Viewer 工作区上传统一使用 `files` 多文件字段，一次最多上传 50 个文件，批量上传失败时清理本次已写入文件。 - 聊天附件新增 MinIO tmp 临时上传、可选 PDF/图片解析、确认后加入线程附件的流程；前端改为弹窗内上传、解析与确认。 +- 修复智能体对话上传透明 PNG 后图片失真的问题：多模态图片处理在导出 RGB 前会先按白底合成 alpha 通道，避免透明像素中的隐藏颜色被直接转为可见像素；交付物预览优先按文件头识别 MIME，避免 `.jpg` 文件名包裹 PNG 内容时前端按错误格式加载；Agent run 输入消息会持久化为 `multimodal_image`，刷新历史后仍能显示用户上传图片。 - 优化智能体对话页细节：状态面板隐藏空 section，待办名称限制为 20 个中文汉字以内，模型选择器展示供应商名称，并收紧附件状态标签与文件编辑浮动操作样式。 - 标准化 Agent run/SSE 执行链路：run 创建时持久化输入消息并提交后入队，worker 统一写入 Redis Stream envelope，SSE 输出 `event/data/id`、心跳注释、`Last-Event-ID` 回放和终止 `end` 事件；前端强制使用 run API 并支持 ask_user_question 中断后以 resume run 恢复；事件 envelope 构造收敛到统一 helper，前端优先使用 envelope 一级 `thread_id` 路由。 - Agent run SSE 新增 `verbose=false` 精简模式：默认仍返回完整事件载荷；精简模式仅在 SSE 输出前重建最小 payload，跳过 `metadata` 和空 `yuxi.agent_state`，将同一 data 内的 `request_id` 外提为单个字段，移除 chunk 中重复的 `meta`、`metadata`、`thread_id`、`response`、空 `namespace` 和图片 base64 等调试字段，保留消息增量、工具调用、工具结果、非空 Agent state、终止状态和 SSE 游标，前端订阅默认使用精简模式。 diff --git a/web/src/components/AgentChatComponent.vue b/web/src/components/AgentChatComponent.vue index ac5771a3..a218143f 100644 --- a/web/src/components/AgentChatComponent.vue +++ b/web/src/components/AgentChatComponent.vue @@ -1283,12 +1283,68 @@ const historyConversations = computed(() => { return MessageProcessor.convertServerHistoryToMessages(currentThreadMessages.value) }) +function getMessageRequestId(message) { + const metadataRequestId = message?.extra_metadata?.request_id + if (typeof metadataRequestId === 'string' && metadataRequestId.trim()) return metadataRequestId.trim() + if (message?.type === 'human' && typeof message.id === 'string' && message.id.trim()) { + return message.id.trim() + } + return null +} + +function mergeLocalImageFields(message, localMessage) { + if (!localMessage?.image_content || message?.image_content) return message + return { + ...message, + message_type: localMessage.message_type || message.message_type, + image_content: localMessage.image_content, + extra_metadata: message.extra_metadata || {} + } +} + +function mergeOngoingUserMessageIntoHistory(historyConvs, ongoingMessages) { + if (!Array.isArray(historyConvs) || !historyConvs.length || !Array.isArray(ongoingMessages)) { + return { historyConvs, ongoingMessages } + } + + const firstOngoingMessage = ongoingMessages[0] + if (!firstOngoingMessage || firstOngoingMessage.type !== 'human') { + return { historyConvs, ongoingMessages } + } + + const lastHistoryConv = historyConvs[historyConvs.length - 1] + const historyMessages = Array.isArray(lastHistoryConv?.messages) ? lastHistoryConv.messages : [] + const historyHumanIndex = historyMessages.findIndex((message) => message?.type === 'human') + if (historyHumanIndex === -1) return { historyConvs, ongoingMessages } + + const historyHuman = historyMessages[historyHumanIndex] + const historyRequestId = getMessageRequestId(historyHuman) + const ongoingRequestId = getMessageRequestId(firstOngoingMessage) + if (!historyRequestId || !ongoingRequestId || historyRequestId !== ongoingRequestId) { + return { historyConvs, ongoingMessages } + } + + const patchedHistoryHuman = mergeLocalImageFields(historyHuman, firstOngoingMessage) + if (patchedHistoryHuman === historyHuman) { + return { historyConvs, ongoingMessages: ongoingMessages.slice(1) } + } + + const patchedHistoryMessages = [...historyMessages] + patchedHistoryMessages[historyHumanIndex] = patchedHistoryHuman + const patchedHistoryConvs = [...historyConvs] + patchedHistoryConvs[historyConvs.length - 1] = { + ...lastHistoryConv, + messages: patchedHistoryMessages + } + return { historyConvs: patchedHistoryConvs, ongoingMessages: ongoingMessages.slice(1) } +} + const conversations = computed(() => { const historyConvs = historyConversations.value - const mergedOngoingMessages = stripDuplicatedOngoingHumanMessage( - historyConvs, - onGoingConvMessages.value - ) + const { + historyConvs: mergedHistoryConvs, + ongoingMessages: mergedOngoingMessages + } = mergeOngoingUserMessageIntoHistory(historyConvs, onGoingConvMessages.value) // 如果有进行中的消息且线程状态显示正在流式处理，添加进行中的对话 if (mergedOngoingMessages.length > 0) { @@ -1296,9 +1352,9 @@ const conversations = computed(() => { messages: mergedOngoingMessages, status: 'streaming' } - return [...historyConvs, onGoingConv] + return [...mergedHistoryConvs, onGoingConv] } - return historyConvs + return mergedHistoryConvs }) const conversationRows = computed(() => { @@ -1391,49 +1447,6 @@ const buildOptimisticHumanMessage = ({ return message } -const getMessageRequestId = (message) => { - if (!message || typeof message !== 'object') return null - - const metadataRequestId = message.extra_metadata?.request_id - if (typeof metadataRequestId === 'string' && metadataRequestId.trim()) { - return metadataRequestId.trim() - } - - if (message.type === 'human' && typeof message.id === 'string' && message.id.trim()) { - return message.id.trim() - } - - return null -} - -// 历史消息已落库时，ongoing 里仍会保留当前轮的本地 user message； -// 切回线程后按 request_id 去掉这条重复消息，只保留仍在流式更新的部分。 -const stripDuplicatedOngoingHumanMessage = (historyConvs, ongoingMessages) => { - if (!Array.isArray(historyConvs) || !historyConvs.length || !Array.isArray(ongoingMessages)) { - return ongoingMessages - } - - const firstOngoingMessage = ongoingMessages[0] - if (!firstOngoingMessage || firstOngoingMessage.type !== 'human') { - return ongoingMessages - } - - const lastHistoryConv = historyConvs[historyConvs.length - 1] - const historyMessages = Array.isArray(lastHistoryConv?.messages) ? lastHistoryConv.messages : [] - const lastHistoryHuman = historyMessages.find((message) => message?.type === 'human') - if (!lastHistoryHuman) { - return ongoingMessages - } - - const historyRequestId = getMessageRequestId(lastHistoryHuman) - const ongoingRequestId = getMessageRequestId(firstOngoingMessage) - if (!historyRequestId || !ongoingRequestId || historyRequestId !== ongoingRequestId) { - return ongoingMessages - } - - return ongoingMessages.slice(1) -} - // 发送 runs 前先在前端插入一条用户消息，避免等待 worker 轮询后消息才出现。 const insertOptimisticHumanMessage = ( threadState, diff --git a/web/src/components/AgentInputArea.vue b/web/src/components/AgentInputArea.vue index b692f6f8..be3fcb12 100644 --- a/web/src/components/AgentInputArea.vue +++ b/web/src/components/AgentInputArea.vue @@ -23,27 +23,7 @@
- - -
- -
@@ -124,12 +104,6 @@ const currentImage = ref(null) const placeholder = '问点什么？使用 @ 可以提及哦~' const previewAttachments = computed(() => normalizeAttachmentPreviews(props.attachments)) -const previewImageAttachments = computed(() => - previewAttachments.value.filter((attachment) => attachment.isImage && attachment.previewUrl) -) -const previewFileAttachments = computed(() => - previewAttachments.value.filter((attachment) => !attachment.isImage || !attachment.previewUrl) -) const updateValue = (val) => { emit('update:modelValue', val) @@ -213,23 +187,6 @@ defineExpose({ gap: 8px; } -.attachment-preview-image { - position: relative; - width: 80px; - height: 80px; - border-radius: 12px; - border: 1px solid var(--gray-150); - background: var(--gray-25); - overflow: hidden; -} - -.attachment-image-thumb { - width: 100%; - height: 100%; - object-fit: cover; - display: block; -} - .attachment-file-card { position: relative; display: flex; @@ -281,8 +238,8 @@ defineExpose({ position: absolute; top: 6px; right: 6px; - width: 24px; - height: 24px; + width: 20px; + height: 20px; border: none; border-radius: 50%; display: inline-flex; diff --git a/web/src/components/AgentMessageComponent.vue b/web/src/components/AgentMessageComponent.vue index 09c0a71c..4af9226c 100644 --- a/web/src/components/AgentMessageComponent.vue +++ b/web/src/components/AgentMessageComponent.vue @@ -1,9 +1,9 @@