From a4f425dbf93f76805f25907ef4b12edd08bd2bcb Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Tue, 27 Jan 2026 15:42:48 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E4=B8=8A=E4=BC=A0=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=9F=BA=E4=BA=8EURL=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E5=92=8C=E5=A4=84=E7=90=86=E5=8A=9F=E8=83=BD=EF=BC=8C=E9=9C=80?= =?UTF-8?q?=E8=A6=81=E9=85=8D=E7=BD=AE=E7=99=BD=E5=90=8D=E5=8D=95=E7=8E=AF?= =?UTF-8?q?=E5=A2=83=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 引入新的端点来获取URL内容并转换为Markdown格式。 - 在环境变量中添加URL白名单配置以确保安全。 - 增强文件上传模态框以支持URL输入和URL批量处理。 - 更新文件处理逻辑以区分文件和URL。 - 改进UI组件以适应新的URL处理功能。 - 添加用于URL验证和获取的工具函数。 --- .env.template | 3 + docs/latest/advanced/document-processing.md | 18 +- pyproject.toml | 1 + server/routers/knowledge_router.py | 87 +++- src/knowledge/base.py | 25 +- src/knowledge/indexing.py | 40 +- src/knowledge/utils/kb_utils.py | 56 ++- src/knowledge/utils/url_fetcher.py | 131 ++++++ src/knowledge/utils/url_validator.py | 86 ++++ web/src/apis/knowledge_api.js | 13 + web/src/components/FileTable.vue | 16 +- web/src/components/FileUploadModal.vue | 416 +++++++++++++++++++- web/src/utils/file_utils.js | 13 +- web/src/views/DataBaseInfoView.vue | 5 +- 14 files changed, 875 insertions(+), 35 deletions(-) create mode 100644 src/knowledge/utils/url_fetcher.py create mode 100644 src/knowledge/utils/url_validator.py diff --git a/.env.template b/.env.template index b258f0ba..71b296f3 100644 --- a/.env.template +++ b/.env.template @@ -25,5 +25,8 @@ TAVILY_API_KEY= # 获取搜索服务的 api key 请访问 https://app.tavily.co # YUXI_SUPER_ADMIN_NAME= # YUXI_SUPER_ADMIN_PASSWORD= +# # URL Whitelist (comma-separated domains/IPs, empty to disable URL parsing) +# YUXI_URL_WHITELIST=github.com,docs.example.com,gitlab.example.com,127.0.0.1 + # # MinerU # MINERU_API_KEY= \ No newline at end of file diff --git a/docs/latest/advanced/document-processing.md b/docs/latest/advanced/document-processing.md index 8681e721..a84740d7 100644 --- a/docs/latest/advanced/document-processing.md +++ b/docs/latest/advanced/document-processing.md @@ -17,6 +17,10 @@ - **电子表格**: `.csv`, `.xls`, `.xlsx` - **JSON 数据**: `.json` +::: tip 图片显示 +文档中的图片会自动上传到对象存储并替换为可访问的 URL。但是如果想要在外部正常显示图片，需要配置 `HOST_IP` 环境变量，将其设置为您的服务器 IP 地址。 +::: + ### 图像格式（需要 OCR） - **常见图片**: `.jpg`, `.jpeg`, `.png`, `.bmp`, `.tiff`, `.tif`, `.gif`, `.webp` @@ -28,10 +32,20 @@ - 优先处理名为 `full.md` 的文件，否则使用第一个 `.md` 文件 - 支持图片目录的智能识别（`images/`、`../images/` 等） -::: tip 图片显示 -文档中的图片会自动上传到对象存储并替换为可访问的 URL。但是如果想要在外部正常显示图片，需要配置 `HOST_IP` 环境变量，将其设置为您的服务器 IP 地址。 +### URL 网页内容 +- **网页链接**: `http://` 或 `https://` + - 自动抓取网页 HTML 内容并转换为 Markdown + - **白名单机制**: 出于安全考虑，必须配置环境变量 `YUXI_URL_WHITELIST` 才能使用此功能 + - **内网保护**: 默认禁止抓取私有 IP 地址（如 127.0.0.1, 192.168.x.x） + - **去重机制**: 自动检测 URL 是否已存在，以及内容 Hash 是否重复 + +::: tip URL 白名单配置 +在 `.env` 文件中配置允许抓取的域名列表，用逗号分隔。支持通配符。 +例如：`YUXI_URL_WHITELIST=github.com,*.wikipedia.org,docs.python.org` ::: + + ## 快速配置 ### 1. 基础 OCR (RapidOCR) diff --git a/pyproject.toml b/pyproject.toml index 60fcc6ee..8f5c0eef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "llama-index>=0.14", "llama-index-readers-file>=0.4.7", "markdownify>=1.1.0", + "readability-lxml>=0.8.1", "mcp>=1.20", "neo4j>=5.28.1", "networkx>=3.5", diff --git a/server/routers/knowledge_router.py b/server/routers/knowledge_router.py index 16b52a3e..1677aad3 100644 --- a/server/routers/knowledge_router.py +++ b/server/routers/knowledge_router.py @@ -269,9 +269,9 @@ async def add_documents( "qa_separator": params.get("qa_separator", ""), } - # 禁止 URL 解析与入库 + # URL 解析与入库（需白名单验证） if content_type == "url": - raise HTTPException(status_code=400, detail="URL 文档上传与解析已禁用") + raise HTTPException(status_code=400, detail="URL 处理方式已变更，请使用 fetch-url 接口先获取内容") # 安全检查：验证文件路径 if content_type == "file": @@ -640,9 +640,14 @@ async def download_document(db_id: str, doc_id: str, request: Request, current_u file_info = await knowledge_base.get_file_basic_info(db_id, doc_id) file_meta = file_info.get("meta", {}) - # 获取文件路径和文件名 + # 获取文件类型、路径和文件名 + file_type = file_meta.get("file_type", "file") file_path = file_meta.get("path", "") filename = file_meta.get("filename", "file") + + # URL 类型文件没有原始文件可下载 + if file_type == "url": + raise HTTPException(status_code=400, detail="URL 类型文件不支持下载原始文件") logger.debug(f"File path from database: {file_path}") logger.debug(f"Original filename from database: {filename}") @@ -1088,6 +1093,82 @@ async def move_document( raise HTTPException(status_code=500, detail=str(e)) +@knowledge.post("/files/fetch-url") +async def fetch_url( + url: str = Body(..., embed=True), + db_id: str | None = Body(None, embed=True), + current_user: User = Depends(get_admin_user), +): + """ + 抓取 URL 内容并上传到 MinIO + """ + logger.debug(f"Fetching URL: {url} for db_id: {db_id}") + try: + from src.knowledge.utils.url_fetcher import fetch_url_content + from src.storage.minio import get_minio_client + from src.knowledge.utils import calculate_content_hash + + # 1. 下载内容 (包含白名单校验、大小限制、类型检查) + content_bytes, final_url = await fetch_url_content(url) + + # 2. 计算 Hash + content_hash = await calculate_content_hash(content_bytes) + + # 检查是否已存在相同内容的文件 + if db_id: + file_exists = await knowledge_base.file_existed_in_db(db_id, content_hash) + if file_exists: + raise HTTPException( + status_code=409, + detail="数据库中已经存在了相同内容文件", + ) + + # 3. 上传到 MinIO + minio_client = get_minio_client() + # 确保存储 bucket 存在 + bucket_name = "kb-html-archives" + await asyncio.to_thread(minio_client.ensure_bucket_exists, bucket_name) + + # 如果没有提供 db_id，使用 default + folder = db_id if db_id else "default" + object_name = f"{folder}/{content_hash}.html" + + upload_result = await minio_client.aupload_file( + bucket_name=bucket_name, + object_name=object_name, + data=content_bytes, + content_type="text/html", + ) + + # 检测同名文件（URL即为文件名） + same_name_files = [] + has_same_name = False + if db_id: + same_name_files = await knowledge_base.get_same_name_files(db_id, url) + has_same_name = len(same_name_files) > 0 + + return { + "status": "success", + "file_path": upload_result.url, + "minio_url": upload_result.url, + "content_hash": content_hash, + "filename": url, # 原始 URL 作为文件名 + "final_url": final_url, + "size": len(content_bytes), + "has_same_name": has_same_name, + "same_name_files": same_name_files, + } + + except HTTPException: + raise + except ValueError as e: + logger.warning(f"URL fetch validation failed: {e}") + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Failed to fetch URL {url}: {e}, {traceback.format_exc()}") + raise HTTPException(status_code=500, detail=f"Failed to fetch URL: {str(e)}") + + @knowledge.post("/files/upload") async def upload_file( file: UploadFile = File(...), diff --git a/src/knowledge/base.py b/src/knowledge/base.py index 83e747cc..c9cadd70 100644 --- a/src/knowledge/base.py +++ b/src/knowledge/base.py @@ -262,14 +262,27 @@ class KnowledgeBase(ABC): self._add_to_processing_queue(file_id) try: - from src.knowledge.indexing import process_file_to_markdown + # Determine processing function based on content type + content_type = file_meta.get("processing_params", {}).get("content_type", "file") - # Prepare params - params = file_meta.get("processing_params", {}) or {} - params["db_id"] = db_id + if content_type == "url": + from src.knowledge.indexing import process_url_to_markdown - # Process to Markdown - markdown_content = await process_file_to_markdown(file_path, params=params) + # Prepare params + params = file_meta.get("processing_params", {}) or {} + params["db_id"] = db_id + + # Process URL to Markdown + markdown_content = await process_url_to_markdown(file_path, params=params) + else: + from src.knowledge.indexing import process_file_to_markdown + + # Prepare params + params = file_meta.get("processing_params", {}) or {} + params["db_id"] = db_id + + # Process file to Markdown + markdown_content = await process_file_to_markdown(file_path, params=params) # Save Markdown to MinIO markdown_file_path = await self._save_markdown_to_minio(db_id, file_id, markdown_content) diff --git a/src/knowledge/indexing.py b/src/knowledge/indexing.py index 036d05d5..6c50ece3 100644 --- a/src/knowledge/indexing.py +++ b/src/knowledge/indexing.py @@ -19,6 +19,7 @@ from langchain_community.document_loaders import ( UnstructuredWordDocumentLoader, ) from langchain_text_splitters import RecursiveCharacterTextSplitter +from markdownify import markdownify as md_convert from src.knowledge.utils import calculate_content_hash from src.storage.minio import get_minio_client @@ -712,4 +713,41 @@ def _replace_image_links(markdown_content: str, images: list[dict]) -> str: async def process_url_to_markdown(url: str, params: dict | None = None) -> str: - raise NotImplementedError("URL 解析功能已禁用") + """ + Fetch a URL and convert its content to Markdown. + + Args: + url: The URL to fetch. + params: Optional parameters (unused, kept for API compatibility). + + Returns: + The Markdown content of the URL. + """ + logger.info(f"Fetching URL: {url}") + + try: + import httpx + # 使用异步 HTTP 客户端获取页面 + async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client: + response = await client.get(url, headers={"User-Agent": "Mozilla/5.0"}) + response.raise_for_status() + html_content = response.text + + # 使用 readability 提取正文 HTML + from readability import Document + + doc = Document(html_content) + body_html = doc.summary() + + # 转换为 Markdown + markdown_content = md_convert(body_html, heading_style="atx") + + logger.info(f"Successfully converted URL to Markdown: {url}") + return markdown_content + + except httpx.HTTPError as e: + logger.error(f"Failed to fetch URL {url}: {e}") + raise ValueError(f"Failed to fetch URL: {e}") + except Exception as e: + logger.error(f"Failed to process URL {url}: {e}") + raise ValueError(f"Failed to process URL: {e}") diff --git a/src/knowledge/utils/kb_utils.py b/src/knowledge/utils/kb_utils.py index 18b40765..370cae1b 100644 --- a/src/knowledge/utils/kb_utils.py +++ b/src/knowledge/utils/kb_utils.py @@ -188,6 +188,51 @@ async def prepare_item_metadata(item: str, content_type: str, db_id: str, params db_id: 数据库ID params: 处理参数，可选 """ + # 检查是否有预处理信息 (针对 URL 转 HTML 文件的情况) + if params and "_preprocessed_map" in params and item in params["_preprocessed_map"]: + pre_info = params["_preprocessed_map"][item] + + # 使用预处理信息 + filename = pre_info.get("filename", item) # 通常是原始 URL + + # 截断文件名以适应数据库限制 (512 chars)，保留部分后缀信息如果可能 + if len(filename) > 500: + filename_display = filename[:400] + "..." + filename[-90:] + else: + filename_display = filename + + file_type = "html" # 强制转换为 html 类型，以便后续作为文件处理 + item_path = pre_info["path"] # MinIO path + content_hash = pre_info["content_hash"] + + # 使用 item(url) 生成 ID，保证同一 URL 即使多次添加 ID 也不同（配合 time） + # 或者我们应该基于 hash？不，基于 time 更符合上传逻辑 + file_id = f"file_{hashstr(item + str(time.time()), 6)}" + + metadata = { + "database_id": db_id, + "filename": filename_display, + "path": item_path, + "file_type": file_type, + "status": "processing", + "created_at": utc_isoformat(), + "file_id": file_id, + "content_hash": content_hash, + "parent_id": params.get("parent_id"), + } + + if params: + # 移除内部参数以免污染 metadata + safe_params = params.copy() + safe_params.pop("_preprocessed_map", None) + # 覆盖 content_type 为 file，确保后续解析走文件流程（MinIO 下载 -> HTML 解析） + # 而不是再次尝试作为 URL 抓取 + safe_params["content_type"] = "file" + safe_params["original_source"] = item # 保存完整 URL 到 JSON 字段，避免数据库字段长度限制 + metadata["processing_params"] = safe_params + + return metadata + if content_type == "file": # 检测是否是MinIO URL还是本地文件路径 if is_minio_url(item): @@ -243,8 +288,17 @@ async def prepare_item_metadata(item: str, content_type: str, db_id: str, params # 生成文件ID file_id = f"file_{hashstr(str(item_path) + str(time.time()), 6)}" + elif content_type == "url": + # URL 处理 + filename = item # 使用完整 URL 作为文件名 + filename_display = item + file_type = "url" + item_path = item + content_hash = None # URL 没有 content_hash + file_id = f"url_{hashstr(item + str(time.time()), 6)}" + else: - raise ValueError("URL 元数据生成已禁用") + raise ValueError(f"Unsupported content_type: {content_type}") metadata = { "database_id": db_id, diff --git a/src/knowledge/utils/url_fetcher.py b/src/knowledge/utils/url_fetcher.py new file mode 100644 index 00000000..c2021456 --- /dev/null +++ b/src/knowledge/utils/url_fetcher.py @@ -0,0 +1,131 @@ +import httpx +from urllib.parse import urlparse +import socket +import ipaddress +from src.knowledge.utils.url_validator import validate_url, is_url_parsing_enabled +from src.utils import logger + +# 最大允许下载大小 (例如 10MB) +MAX_DOWNLOAD_SIZE = 10 * 1024 * 1024 +# 允许的 Content-Type +ALLOWED_CONTENT_TYPES = ["text/html", "application/xhtml+xml"] + +async def is_private_ip(hostname: str) -> bool: + """Check if the hostname resolves to a private IP address.""" + import asyncio + try: + # Resolve hostname to IP in a separate thread to avoid blocking the event loop + ip_list = await asyncio.to_thread(socket.getaddrinfo, hostname, None) + for item in ip_list: + ip_addr = item[4][0] + ip_obj = ipaddress.ip_address(ip_addr) + if ip_obj.is_private or ip_obj.is_loopback or ip_obj.is_link_local: + return True + return False + except Exception as e: + logger.warning(f"Failed to resolve hostname {hostname}: {e}") + # If resolution fails, assume it's unsafe or let the connection fail naturally, + # but to be safe we can return True to block it if strict mode is preferred. + # For now, we return False assuming standard DNS failure handling. + return False + +async def fetch_url_content(url: str, max_size: int = MAX_DOWNLOAD_SIZE) -> tuple[bytes, str]: + """ + Fetch URL content with security checks (size limit, content type, private IP blocking). + + Args: + url: The URL to fetch. + max_size: Maximum allowed size in bytes. + + Returns: + tuple: (content_bytes, final_url) + + Raises: + ValueError: If validation fails or download error occurs. + """ + if not is_url_parsing_enabled(): + raise ValueError("URL parsing feature is disabled") + + # Initial validation + is_valid, error_msg = validate_url(url) + if not is_valid: + raise ValueError(f"Invalid URL: {error_msg}") + + # Parse URL to check IP + parsed_url = urlparse(url) + if await is_private_ip(parsed_url.hostname): + raise ValueError("Access to private IP addresses is forbidden") + + current_url = url + redirect_count = 0 + max_redirects = 5 + + # We handle redirects manually to check each target URL against whitelist/private IP + try: + async with httpx.AsyncClient(timeout=30.0, follow_redirects=False) as client: + while True: + logger.info(f"Fetching URL: {current_url}") + + # Request headers + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" + } + + # Stream the response to check headers before downloading body + async with client.stream("GET", current_url, headers=headers) as response: + # Handle Redirects + if response.status_code in (301, 302, 303, 307, 308): + if redirect_count >= max_redirects: + raise ValueError("Too many redirects") + + redirect_count += 1 + location = response.headers.get("Location") + if not location: + raise ValueError("Redirect response missing Location header") + + # Handle relative redirects + if location.startswith("/"): + parsed_current = urlparse(current_url) + current_url = f"{parsed_current.scheme}://{parsed_current.netloc}{location}" + elif not location.startswith("http"): + # Handle relative path without /? or other weird cases, or assume absolute + parsed_current = urlparse(current_url) + # simple join + from urllib.parse import urljoin + current_url = urljoin(current_url, location) + else: + current_url = location + + # Validate the new URL + is_valid, error_msg = validate_url(current_url) + if not is_valid: + raise ValueError(f"Redirected to invalid URL: {error_msg}") + + parsed_new = urlparse(current_url) + if await is_private_ip(parsed_new.hostname): + raise ValueError("Redirected to private IP address") + + continue # Start new request + + response.raise_for_status() + + # Check Content-Type + content_type = response.headers.get("Content-Type", "").lower() + if not any(allowed in content_type for allowed in ALLOWED_CONTENT_TYPES): + raise ValueError(f"Unsupported Content-Type: {content_type}. Only HTML is supported.") + + # Download content with size limit + content = bytearray() + async for chunk in response.aiter_bytes(): + content.extend(chunk) + if len(content) > max_size: + raise ValueError(f"Content size exceeds limit of {max_size} bytes") + + return bytes(content), current_url + + except httpx.HTTPError as e: + logger.error(f"HTTP error fetching {url}: {e}") + raise ValueError(f"Failed to fetch URL: {e}") + except Exception as e: + logger.error(f"Error fetching {url}: {e}") + raise ValueError(f"Error fetching URL: {str(e)}") diff --git a/src/knowledge/utils/url_validator.py b/src/knowledge/utils/url_validator.py new file mode 100644 index 00000000..e151d97c --- /dev/null +++ b/src/knowledge/utils/url_validator.py @@ -0,0 +1,86 @@ +"""URL validation utilities for whitelist-based URL parsing.""" +import os +from urllib.parse import urlparse + +# Environment variable name for URL whitelist +YUXI_URL_WHITELIST_ENV = "YUXI_URL_WHITELIST" + + +def _get_whitelist() -> list[str]: + """Get the URL whitelist from environment variables.""" + whitelist_str = os.environ.get(YUXI_URL_WHITELIST_ENV, "") + if not whitelist_str: + return [] + # Split by comma and clean up whitespace + return [item.strip() for item in whitelist_str.split(",") if item.strip()] + + +def validate_url(url: str) -> tuple[bool, str]: + """ + Validate if a URL is in the whitelist. + + Args: + url: The URL to validate. + + Returns: + A tuple of (is_valid, error_message). + If is_valid is True, error_message will be empty. + """ + if not url: + return False, "URL cannot be empty" + + # Parse the URL + try: + parsed = urlparse(url) + except Exception as e: + return False, f"Invalid URL format: {e}" + + # Check if URL has a valid scheme + if not parsed.scheme: + return False, "URL must start with http:// or https://" + + if parsed.scheme not in ("http", "https"): + return False, "URL must use HTTP or HTTPS protocol" + + # Get the hostname + hostname = parsed.hostname + if not hostname: + return False, "Invalid URL: no hostname found" + + # Get the whitelist + whitelist = _get_whitelist() + + # If whitelist is empty, URL parsing is disabled + if not whitelist: + return False, "URL parsing feature is disabled" + + # Check if hostname is in whitelist + for allowed in whitelist: + # Handle wildcard patterns like *.example.com + if allowed.startswith("*."): + domain = allowed[2:] + # Check if hostname ends with the domain (or matches exactly) + if hostname == domain or hostname.endswith(f".{domain}"): + return True, "" + else: + # Exact match or subdomain match + if hostname == allowed or hostname.endswith(f".{allowed}"): + return True, "" + + return False, f"Domain '{hostname}' is not in the whitelist" + + +def is_url_parsing_enabled() -> bool: + """Check if URL parsing is enabled (whitelist is configured).""" + whitelist = _get_whitelist() + return len(whitelist) > 0 + + +def get_whitelist_info() -> dict: + """Get information about the current whitelist configuration.""" + whitelist = _get_whitelist() + return { + "enabled": len(whitelist) > 0, + "domains": whitelist, + "count": len(whitelist), + } diff --git a/web/src/apis/knowledge_api.js b/web/src/apis/knowledge_api.js index 12230693..8c23230b 100644 --- a/web/src/apis/knowledge_api.js +++ b/web/src/apis/knowledge_api.js @@ -259,6 +259,19 @@ export const queryApi = { // ============================================================================= export const fileApi = { + /** + * 抓取 URL 内容 + * @param {string} url - 目标 URL + * @param {string} dbId - 知识库 ID + * @returns {Promise} - 抓取结果 + */ + fetchUrl: async (url, dbId = null) => { + return apiAdminPost('/api/knowledge/files/fetch-url', { + url, + db_id: dbId + }) + }, + /** * 上传文件 * @param {File} file - 文件对象 diff --git a/web/src/components/FileTable.vue b/web/src/components/FileTable.vue index ea3c835b..c8645954 100644 --- a/web/src/components/FileTable.vue +++ b/web/src/components/FileTable.vue @@ -24,6 +24,13 @@ > 上传文件夹 + + 解析 URL + @@ -342,6 +349,7 @@ @click="handleDownloadFile(record)" :disabled=" lock || + record.file_type === 'url' || !['done', 'indexed', 'parsed', 'error_indexing'].includes(record.status) " > @@ -441,7 +449,8 @@ import { Search, Filter, ArrowUpDown, - ChevronDown + ChevronDown, + Link } from 'lucide-vue-next' const store = useDatabaseStore() @@ -831,7 +840,10 @@ const buildFileTree = (fileList) => { const normalizedName = file.filename.replace(/\\/g, '/') const parts = normalizedName.split('/') - if (parts.length === 1) { + // 检测是否是 URL（URL 不应该被解析为文件夹层级） + const isUrl = file.filename.startsWith('http://') || file.filename.startsWith('https://') + + if (isUrl || parts.length === 1) { // Root item // Check if it's an explicit folder that should merge with an existing implicit one? if (item.is_folder) { diff --git a/web/src/components/FileUploadModal.vue b/web/src/components/FileUploadModal.vue index 41024c48..ddd85e19 100644 --- a/web/src/components/FileUploadModal.vue +++ b/web/src/components/FileUploadModal.vue @@ -12,7 +12,11 @@ type="primary" @click="chunkData" :loading="chunkLoading" - :disabled="fileList.length === 0" + :disabled=" + uploadMode === 'url' + ? !urlList.some((i) => i.status === 'success') + : fileList.length === 0 + " > 添加到知识库 @@ -30,13 +34,23 @@ class="custom-segmented" /> +
+ 上传后自动入库 +
-
+
-
-
+
+
存储位置

选择文件保存的目标文件夹

-
+
OCR 引擎 @@ -89,13 +103,11 @@
- -
+ +
-
- 上传后自动入库 -
-
+
入库参数配置
+
@@ -117,7 +129,7 @@
-
+
+ +
+
+ +
+ + 支持批量粘贴，自动过滤空行。 + 需配置白名单，详见文档说明 + + + 加载 URLs + +
+
+
+
+
+ + + +
+
+ {{ item.url }} + {{ item.error }} +
+ + + +
+
+
+ + 输入 URL 后点击加载，系统将自动抓取网页内容 +
+
+
@@ -185,7 +250,17 @@ import { useDatabaseStore } from '@/stores/database' import { ocrApi } from '@/apis/system_api' import { fileApi, documentApi } from '@/apis/knowledge_api' import { CheckCircleFilled, ReloadOutlined } from '@ant-design/icons-vue' -import { FileUp, FolderUp, RotateCw, CircleHelp, Info, Download, Trash2 } from 'lucide-vue-next' +import { + FileUp, + FolderUp, + RotateCw, + CircleHelp, + Info, + Download, + Trash2, + Link, + X +} from 'lucide-vue-next' import { h } from 'vue' import ChunkParamsConfig from '@/components/ChunkParamsConfig.vue' @@ -205,6 +280,10 @@ const props = defineProps({ isFolderMode: { type: Boolean, default: false + }, + mode: { + type: String, + default: 'file' } }) @@ -238,7 +317,7 @@ watch( if (newVal) { selectedFolderId.value = props.currentFolderId isFolderUpload.value = props.isFolderMode - uploadMode.value = props.isFolderMode ? 'folder' : 'file' + uploadMode.value = props.mode || (props.isFolderMode ? 'folder' : 'file') } } ) @@ -343,23 +422,125 @@ const uploadModeOptions = computed(() => [ h(FolderUp, { size: 16, class: 'option-icon' }), h('span', { class: 'option-text' }, '上传文件夹') ]) + }, + { + value: 'url', + label: h('div', { class: 'segmented-option' }, [ + h(Link, { size: 16, class: 'option-icon' }), + h('span', { class: 'option-text' }, '解析 URL') + ]) } ]) watch(uploadMode, (val) => { isFolderUpload.value = val === 'folder' - // 切换模式时清空已选文件，避免混淆 + // 切换模式时清空已选内容，避免混淆 fileList.value = [] sameNameFiles.value = [] + urlList.value = [] + newUrl.value = '' }) // 文件列表 const fileList = ref([]) +// URL 列表 +// Item structure: { url: string, status: 'fetching'|'success'|'error', data: object|null, error: string } +const urlList = ref([]) +const newUrl = ref('') +const fetchingUrls = ref(false) + // 同名文件列表（用于显示提示） const sameNameFiles = ref([]) -// URL相关功能已移除 +// URL 相关功能 +const isValidUrl = (string) => { + try { + const url = new URL(string) + return url.protocol === 'http:' || url.protocol === 'https:' + } catch (_) { + return false + } +} + +const handleFetchUrls = async () => { + const text = newUrl.value + if (!text) return + + const lines = text + .split(/[\r\n]+/) + .map((l) => l.trim()) + .filter((l) => l) + if (lines.length === 0) return + + // 1. 预处理：添加到列表 + const newItems = [] + for (const url of lines) { + if (!isValidUrl(url)) { + continue + } + if (urlList.value.some((u) => u.url === url)) continue + + const item = { url, status: 'pending', data: null, error: '' } + urlList.value.push(item) + newItems.push(item) + } + + if (newItems.length === 0) { + if (lines.length > 0) { + message.warning('没有检测到有效的新 URL') + } + return + } + + newUrl.value = '' // 清空输入框 + fetchingUrls.value = true + + // 2. 并发处理 + // 为避免过多并发请求，可以考虑使用 p-limit 或简单的分批，但此处直接并发 + const processItem = async (item) => { + item.status = 'fetching' + try { + const res = await fileApi.fetchUrl(item.url, databaseId.value) + item.status = 'success' + item.data = res + + // 处理同名文件冲突提示 + if (res.has_same_name && res.same_name_files && res.same_name_files.length > 0) { + // 合并到现有的同名文件列表中，去重 + const existingIds = new Set(sameNameFiles.value.map((f) => f.file_id)) + const newConflicts = res.same_name_files.filter((f) => !existingIds.has(f.file_id)) + sameNameFiles.value.push(...newConflicts) + } + } catch (error) { + console.error('Failed to fetch URL:', error) + item.status = 'error' + + // 特别处理内容重复 (409) + const detail = error.response?.data?.detail || error.message || '' + if (detail.includes('same content') || detail.includes('相同内容')) { + item.error = '内容已存在于知识库中' + } else { + item.error = detail || '加载失败' + } + } + } + + await Promise.all(newItems.map(processItem)) + fetchingUrls.value = false +} + +const removeUrl = (index) => { + urlList.value.splice(index, 1) +} + +const handleUrlKeydown = (e) => { + // Ctrl + Enter 提交 + if (e.key === 'Enter' && e.ctrlKey) { + e.preventDefault() + handleFetchUrls() + } +} // OCR服务健康状态 const ocrHealthStatus = ref({ @@ -790,11 +971,68 @@ const chunkData = async () => { return } - // 验证OCR服务可用性 - if (!validateOcrService()) { + // 验证OCR服务可用性（非 URL 模式下） + if (uploadMode.value !== 'url' && !validateOcrService()) { return } + // URL 模式处理 + if (uploadMode.value === 'url') { + // 过滤出成功的项 + const successfulItems = urlList.value.filter((item) => item.status === 'success' && item.data) + if (successfulItems.length === 0) { + message.error('请添加并等待至少一个 URL 解析成功') + return + } + + try { + store.state.chunkLoading = true + const params = { ...chunkParams.value } + if (autoIndex.value) { + params.auto_index = true + Object.assign(params, indexParams.value) + } + + // 构造 _preprocessed_map 和 items (minio urls) + const items = [] + const preprocessedMap = {} + for (const item of successfulItems) { + // item.data = { file_path: "http://minio...", content_hash: "...", filename: "...", ... } + // 注意：fetch-url 返回的 file_path 其实是 MinIO URL + // 我们需要传递 MinIO URL 给 addDocuments + const minioUrl = item.data.file_path + items.push(minioUrl) + preprocessedMap[minioUrl] = { + path: minioUrl, + content_hash: item.data.content_hash, + filename: item.data.filename, + file_size: item.data.size + } + } + params._preprocessed_map = preprocessedMap + + // 调用 addFiles (file mode) + await store.addFiles({ + items: items, + contentType: 'file', // 重要：这里改为 file，因为我们已经转成了 minio 上的文件 + params, + parentId: selectedFolderId.value + }) + + emit('success') + handleCancel() + urlList.value = [] + newUrl.value = '' + } catch (error) { + console.error('URL 提交失败:', error) + message.error('URL 提交失败: ' + (error.message || '未知错误')) + } finally { + store.state.chunkLoading = false + } + return + } + + // 文件模式处理 const imageExtensions = ['.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif'] // 提取已上传的文件信息 @@ -880,12 +1118,25 @@ const chunkData = async () => { align-items: center; } +.auto-index-toggle { + display: flex; + align-items: center; + padding-right: 4px; + + :deep(.ant-checkbox-wrapper) { + font-size: 13px; + color: var(--gray-600); + font-weight: 500; + } +} + .help-link-btn { color: var(--gray-600); font-size: 13px; display: flex; align-items: center; gap: 6px; + padding: 0; &:hover { color: var(--main-color); @@ -1127,6 +1378,135 @@ const chunkData = async () => { border-radius: 4px; } +/* URL Area */ +.url-area { + flex: 1; + display: flex; + flex-direction: column; + gap: 12px; +} + +.url-input-wrapper { + width: 100%; +} + +.url-actions { + display: flex; + align-items: center; + justify-content: flex-end; + gap: 12px; + margin-top: 8px; +} + +.url-hint { + font-size: 12px; + color: var(--gray-500); + + .warning-text { + color: var(--color-warning-500); + margin-left: 4px; + } +} + +.url-input { + width: 100%; + padding: 10px; +} + +.add-url-btn { + margin-left: 8px; +} + +.url-list { + display: flex; + flex-direction: column; + gap: 8px; + max-height: 300px; + overflow-y: auto; +} + +.url-item { + display: flex; + align-items: center; + gap: 8px; + padding: 8px 12px; + background: var(--gray-50); + border: 1px solid var(--gray-200); + border-radius: 6px; + transition: all 0.2s; + + &:hover { + background: var(--gray-100); + border-color: var(--main-300); + } +} + +.url-icon-wrapper { + display: flex; + align-items: center; + flex-shrink: 0; +} + +.url-icon { + color: var(--main-500); + + &.success { + color: var(--color-success-500); + } + + &.error { + color: var(--color-error-500); + cursor: help; + } + + &.spinning { + animation: spin 1s linear infinite; + color: var(--main-500); + } +} + +.url-content { + flex: 1; + display: flex; + flex-direction: column; + overflow: hidden; +} + +.url-text { + font-size: 13px; + color: var(--gray-700); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.url-error-msg { + font-size: 11px; + color: var(--color-error-500); + margin-top: 2px; +} + +.remove-url-btn { + color: var(--gray-400); + flex-shrink: 0; + + &:hover { + color: var(--color-error-500); + } +} + +.url-empty-tip { + display: flex; + align-items: center; + gap: 8px; + padding: 16px; + background: var(--gray-50); + border: 1px dashed var(--gray-300); + border-radius: 8px; + color: var(--gray-500); + font-size: 13px; +} + /* Conflict Files Panel */ .conflict-files-panel { border: 1px solid var(--gray-200); diff --git a/web/src/utils/file_utils.js b/web/src/utils/file_utils.js index 6e0a33d7..4ebec02a 100644 --- a/web/src/utils/file_utils.js +++ b/web/src/utils/file_utils.js @@ -7,7 +7,8 @@ import { FileExcelFilled, FileImageFilled, FileUnknownFilled, - FilePptFilled + FilePptFilled, + LinkOutlined } from '@ant-design/icons-vue' import { formatRelative, parseToShanghai } from '@/utils/time' @@ -15,6 +16,11 @@ import { formatRelative, parseToShanghai } from '@/utils/time' export const getFileIcon = (filename) => { if (!filename) return FileUnknownFilled + // Check if it's a URL + if (filename.startsWith('http://') || filename.startsWith('https://')) { + return LinkOutlined + } + const extension = filename.toLowerCase().split('.').pop() const iconMap = { @@ -64,6 +70,11 @@ export const getFileIcon = (filename) => { export const getFileIconColor = (filename) => { if (!filename) return '#8c8c8c' + // Check if it's a URL + if (filename.startsWith('http://') || filename.startsWith('https://')) { + return '#1890ff' // Blue for links + } + const extension = filename.toLowerCase().split('.').pop() const colorMap = { diff --git a/web/src/views/DataBaseInfoView.vue b/web/src/views/DataBaseInfoView.vue index eec044f2..3f1e2c41 100644 --- a/web/src/views/DataBaseInfoView.vue +++ b/web/src/views/DataBaseInfoView.vue @@ -14,6 +14,7 @@ :folder-tree="folderTree" :current-folder-id="currentFolderId" :is-folder-mode="isFolderUploadMode" + :mode="addFilesMode" @success="onFileUploadSuccess" /> @@ -314,14 +315,16 @@ const openSearchConfigModal = () => { const addFilesModalVisible = ref(false) const currentFolderId = ref(null) const isFolderUploadMode = ref(false) +const addFilesMode = ref('file') // 标记是否是初次加载 const isInitialLoad = ref(true) // 显示添加文件弹窗 const showAddFilesModal = (options = {}) => { - const { isFolder = false } = options + const { isFolder = false, mode = 'file' } = options isFolderUploadMode.value = isFolder + addFilesMode.value = mode addFilesModalVisible.value = true currentFolderId.value = null // 重置 }