feat: 上传文件新增基于URL获取和处理功能，需要配置白名单环境变量

- 引入新的端点来获取URL内容并转换为Markdown格式。 - 在环境变量中添加URL白名单配置以确保安全。 - 增强文件上传模态框以支持URL输入和URL批量处理。 - 更新文件处理逻辑以区分文件和URL。 - 改进UI组件以适应新的URL处理功能。 - 添加用于URL验证和获取的工具函数。
2026-01-27 15:42:48 +08:00 · 2026-01-27 15:42:48 +08:00 · a4f425dbf9
commit a4f425dbf9
parent 537b798f9a
14 changed files with 875 additions and 35 deletions
--- a/.env.template
+++ b/.env.template
@ -25,5 +25,8 @@ TAVILY_API_KEY=  # 获取搜索服务的 api key 请访问 https://app.tavily.co
 # YUXI_SUPER_ADMIN_NAME=
 # YUXI_SUPER_ADMIN_PASSWORD=

+# # URL Whitelist (comma-separated domains/IPs, empty to disable URL parsing)
+# YUXI_URL_WHITELIST=github.com,docs.example.com,gitlab.example.com,127.0.0.1
+
 # # MinerU
 # MINERU_API_KEY=
--- a/docs/latest/advanced/document-processing.md
+++ b/docs/latest/advanced/document-processing.md
@ -17,6 +17,10 @@
 - **电子表格**: `.csv`, `.xls`, `.xlsx`
 - **JSON 数据**: `.json`

+::: tip 图片显示
+文档中的图片会自动上传到对象存储并替换为可访问的 URL。但是如果想要在外部正常显示图片，需要配置 `HOST_IP` 环境变量，将其设置为您的服务器 IP 地址。
+:::
+
 ### 图像格式（需要 OCR）
 - **常见图片**: `.jpg`, `.jpeg`, `.png`, `.bmp`, `.tiff`, `.tif`, `.gif`, `.webp`

@ -28,10 +32,20 @@
  - 优先处理名为 `full.md` 的文件，否则使用第一个 `.md` 文件
  - 支持图片目录的智能识别（`images/`、`../images/` 等）

-::: tip 图片显示
-文档中的图片会自动上传到对象存储并替换为可访问的 URL。但是如果想要在外部正常显示图片，需要配置 `HOST_IP` 环境变量，将其设置为您的服务器 IP 地址。
+### URL 网页内容
+- **网页链接**: `http://` 或 `https://`
+  - 自动抓取网页 HTML 内容并转换为 Markdown
+  - **白名单机制**: 出于安全考虑，必须配置环境变量 `YUXI_URL_WHITELIST` 才能使用此功能
+  - **内网保护**: 默认禁止抓取私有 IP 地址（如 127.0.0.1, 192.168.x.x）
+  - **去重机制**: 自动检测 URL 是否已存在，以及内容 Hash 是否重复
+
+::: tip URL 白名单配置
+在 `.env` 文件中配置允许抓取的域名列表，用逗号分隔。支持通配符。
+例如：`YUXI_URL_WHITELIST=github.com,*.wikipedia.org,docs.python.org`
 :::

+
+
 ## 快速配置

 ### 1. 基础 OCR (RapidOCR)
--- a/pyproject.toml
+++ b/pyproject.toml
@ -28,6 +28,7 @@ dependencies = [
    "llama-index>=0.14",
    "llama-index-readers-file>=0.4.7",
    "markdownify>=1.1.0",
+    "readability-lxml>=0.8.1",
    "mcp>=1.20",
    "neo4j>=5.28.1",
    "networkx>=3.5",
--- a/server/routers/knowledge_router.py
+++ b/server/routers/knowledge_router.py
@ -269,9 +269,9 @@ async def add_documents(
        "qa_separator": params.get("qa_separator", ""),
    }

-    # 禁止 URL 解析与入库
+    # URL 解析与入库（需白名单验证）
    if content_type == "url":
-        raise HTTPException(status_code=400, detail="URL 文档上传与解析已禁用")
+        raise HTTPException(status_code=400, detail="URL 处理方式已变更，请使用 fetch-url 接口先获取内容")

    # 安全检查：验证文件路径
    if content_type == "file":
@ -640,9 +640,14 @@ async def download_document(db_id: str, doc_id: str, request: Request, current_u
        file_info = await knowledge_base.get_file_basic_info(db_id, doc_id)
        file_meta = file_info.get("meta", {})

-        # 获取文件路径和文件名
+        # 获取文件类型、路径和文件名
+        file_type = file_meta.get("file_type", "file")
        file_path = file_meta.get("path", "")
        filename = file_meta.get("filename", "file")
+
+        # URL 类型文件没有原始文件可下载
+        if file_type == "url":
+            raise HTTPException(status_code=400, detail="URL 类型文件不支持下载原始文件")
        logger.debug(f"File path from database: {file_path}")
        logger.debug(f"Original filename from database: {filename}")

@ -1088,6 +1093,82 @@ async def move_document(
        raise HTTPException(status_code=500, detail=str(e))


+@knowledge.post("/files/fetch-url")
+async def fetch_url(
+    url: str = Body(..., embed=True),
+    db_id: str | None = Body(None, embed=True),
+    current_user: User = Depends(get_admin_user),
+):
+    """
+    抓取 URL 内容并上传到 MinIO
+    """
+    logger.debug(f"Fetching URL: {url} for db_id: {db_id}")
+    try:
+        from src.knowledge.utils.url_fetcher import fetch_url_content
+        from src.storage.minio import get_minio_client
+        from src.knowledge.utils import calculate_content_hash
+
+        # 1. 下载内容 (包含白名单校验、大小限制、类型检查)
+        content_bytes, final_url = await fetch_url_content(url)
+
+        # 2. 计算 Hash
+        content_hash = await calculate_content_hash(content_bytes)
+
+        # 检查是否已存在相同内容的文件
+        if db_id:
+            file_exists = await knowledge_base.file_existed_in_db(db_id, content_hash)
+            if file_exists:
+                raise HTTPException(
+                    status_code=409,
+                    detail="数据库中已经存在了相同内容文件",
+                )
+
+        # 3. 上传到 MinIO
+        minio_client = get_minio_client()
+        # 确保存储 bucket 存在
+        bucket_name = "kb-html-archives"
+        await asyncio.to_thread(minio_client.ensure_bucket_exists, bucket_name)
+
+        # 如果没有提供 db_id，使用 default
+        folder = db_id if db_id else "default"
+        object_name = f"{folder}/{content_hash}.html"
+
+        upload_result = await minio_client.aupload_file(
+            bucket_name=bucket_name,
+            object_name=object_name,
+            data=content_bytes,
+            content_type="text/html",
+        )
+
+        # 检测同名文件（URL即为文件名）
+        same_name_files = []
+        has_same_name = False
+        if db_id:
+            same_name_files = await knowledge_base.get_same_name_files(db_id, url)
+            has_same_name = len(same_name_files) > 0
+
+        return {
+            "status": "success",
+            "file_path": upload_result.url,
+            "minio_url": upload_result.url,
+            "content_hash": content_hash,
+            "filename": url,  # 原始 URL 作为文件名
+            "final_url": final_url,
+            "size": len(content_bytes),
+            "has_same_name": has_same_name,
+            "same_name_files": same_name_files,
+        }
+
+    except HTTPException:
+        raise
+    except ValueError as e:
+        logger.warning(f"URL fetch validation failed: {e}")
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Failed to fetch URL {url}: {e}, {traceback.format_exc()}")
+        raise HTTPException(status_code=500, detail=f"Failed to fetch URL: {str(e)}")
+
+
@knowledge.post("/files/upload")
 async def upload_file(
    file: UploadFile = File(...),
--- a/src/knowledge/base.py
+++ b/src/knowledge/base.py
@ -262,14 +262,27 @@ class KnowledgeBase(ABC):
        self._add_to_processing_queue(file_id)

        try:
-            from src.knowledge.indexing import process_file_to_markdown
+            # Determine processing function based on content type
+            content_type = file_meta.get("processing_params", {}).get("content_type", "file")

-            # Prepare params
-            params = file_meta.get("processing_params", {}) or {}
-            params["db_id"] = db_id
+            if content_type == "url":
+                from src.knowledge.indexing import process_url_to_markdown

-            # Process to Markdown
-            markdown_content = await process_file_to_markdown(file_path, params=params)
+                # Prepare params
+                params = file_meta.get("processing_params", {}) or {}
+                params["db_id"] = db_id
+
+                # Process URL to Markdown
+                markdown_content = await process_url_to_markdown(file_path, params=params)
+            else:
+                from src.knowledge.indexing import process_file_to_markdown
+
+                # Prepare params
+                params = file_meta.get("processing_params", {}) or {}
+                params["db_id"] = db_id
+
+                # Process file to Markdown
+                markdown_content = await process_file_to_markdown(file_path, params=params)

            # Save Markdown to MinIO
            markdown_file_path = await self._save_markdown_to_minio(db_id, file_id, markdown_content)
--- a/src/knowledge/indexing.py
+++ b/src/knowledge/indexing.py
@ -19,6 +19,7 @@ from langchain_community.document_loaders import (
    UnstructuredWordDocumentLoader,
 )
 from langchain_text_splitters import RecursiveCharacterTextSplitter
+from markdownify import markdownify as md_convert

 from src.knowledge.utils import calculate_content_hash
 from src.storage.minio import get_minio_client
@ -712,4 +713,41 @@ def _replace_image_links(markdown_content: str, images: list[dict]) -> str:


 async def process_url_to_markdown(url: str, params: dict | None = None) -> str:
-    raise NotImplementedError("URL 解析功能已禁用")
+    """
+    Fetch a URL and convert its content to Markdown.
+
+    Args:
+        url: The URL to fetch.
+        params: Optional parameters (unused, kept for API compatibility).
+
+    Returns:
+        The Markdown content of the URL.
+    """
+    logger.info(f"Fetching URL: {url}")
+
+    try:
+        import httpx
+        # 使用异步 HTTP 客户端获取页面
+        async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+            response = await client.get(url, headers={"User-Agent": "Mozilla/5.0"})
+            response.raise_for_status()
+            html_content = response.text
+
+        # 使用 readability 提取正文 HTML
+        from readability import Document
+
+        doc = Document(html_content)
+        body_html = doc.summary()
+
+        # 转换为 Markdown
+        markdown_content = md_convert(body_html, heading_style="atx")
+
+        logger.info(f"Successfully converted URL to Markdown: {url}")
+        return markdown_content
+
+    except httpx.HTTPError as e:
+        logger.error(f"Failed to fetch URL {url}: {e}")
+        raise ValueError(f"Failed to fetch URL: {e}")
+    except Exception as e:
+        logger.error(f"Failed to process URL {url}: {e}")
+        raise ValueError(f"Failed to process URL: {e}")
--- a/src/knowledge/utils/kb_utils.py
+++ b/src/knowledge/utils/kb_utils.py
@ -188,6 +188,51 @@ async def prepare_item_metadata(item: str, content_type: str, db_id: str, params
        db_id: 数据库ID
        params: 处理参数，可选
    """
+    # 检查是否有预处理信息 (针对 URL 转 HTML 文件的情况)
+    if params and "_preprocessed_map" in params and item in params["_preprocessed_map"]:
+        pre_info = params["_preprocessed_map"][item]
+
+        # 使用预处理信息
+        filename = pre_info.get("filename", item)  # 通常是原始 URL
+        
+        # 截断文件名以适应数据库限制 (512 chars)，保留部分后缀信息如果可能
+        if len(filename) > 500:
+            filename_display = filename[:400] + "..." + filename[-90:]
+        else:
+            filename_display = filename
+            
+        file_type = "html"  # 强制转换为 html 类型，以便后续作为文件处理
+        item_path = pre_info["path"]  # MinIO path
+        content_hash = pre_info["content_hash"]
+
+        # 使用 item(url) 生成 ID，保证同一 URL 即使多次添加 ID 也不同（配合 time）
+        # 或者我们应该基于 hash？不，基于 time 更符合上传逻辑
+        file_id = f"file_{hashstr(item + str(time.time()), 6)}"
+
+        metadata = {
+            "database_id": db_id,
+            "filename": filename_display,
+            "path": item_path,
+            "file_type": file_type,
+            "status": "processing",
+            "created_at": utc_isoformat(),
+            "file_id": file_id,
+            "content_hash": content_hash,
+            "parent_id": params.get("parent_id"),
+        }
+
+        if params:
+            # 移除内部参数以免污染 metadata
+            safe_params = params.copy()
+            safe_params.pop("_preprocessed_map", None)
+            # 覆盖 content_type 为 file，确保后续解析走文件流程（MinIO 下载 -> HTML 解析）
+            # 而不是再次尝试作为 URL 抓取
+            safe_params["content_type"] = "file"
+            safe_params["original_source"] = item  # 保存完整 URL 到 JSON 字段，避免数据库字段长度限制
+            metadata["processing_params"] = safe_params
+
+        return metadata
+
    if content_type == "file":
        # 检测是否是MinIO URL还是本地文件路径
        if is_minio_url(item):
@ -243,8 +288,17 @@ async def prepare_item_metadata(item: str, content_type: str, db_id: str, params
        # 生成文件ID
        file_id = f"file_{hashstr(str(item_path) + str(time.time()), 6)}"

+    elif content_type == "url":
+        # URL 处理
+        filename = item  # 使用完整 URL 作为文件名
+        filename_display = item
+        file_type = "url"
+        item_path = item
+        content_hash = None  # URL 没有 content_hash
+        file_id = f"url_{hashstr(item + str(time.time()), 6)}"
+
    else:
-        raise ValueError("URL 元数据生成已禁用")
+        raise ValueError(f"Unsupported content_type: {content_type}")

    metadata = {
        "database_id": db_id,
--- a/src/knowledge/utils/url_fetcher.py
+++ b/src/knowledge/utils/url_fetcher.py
@ -0,0 +1,131 @@
+import httpx
+from urllib.parse import urlparse
+import socket
+import ipaddress
+from src.knowledge.utils.url_validator import validate_url, is_url_parsing_enabled
+from src.utils import logger
+
+# 最大允许下载大小 (例如 10MB)
+MAX_DOWNLOAD_SIZE = 10 * 1024 * 1024
+# 允许的 Content-Type
+ALLOWED_CONTENT_TYPES = ["text/html", "application/xhtml+xml"]
+
+async def is_private_ip(hostname: str) -> bool:
+    """Check if the hostname resolves to a private IP address."""
+    import asyncio
+    try:
+        # Resolve hostname to IP in a separate thread to avoid blocking the event loop
+        ip_list = await asyncio.to_thread(socket.getaddrinfo, hostname, None)
+        for item in ip_list:
+            ip_addr = item[4][0]
+            ip_obj = ipaddress.ip_address(ip_addr)
+            if ip_obj.is_private or ip_obj.is_loopback or ip_obj.is_link_local:
+                return True
+        return False
+    except Exception as e:
+        logger.warning(f"Failed to resolve hostname {hostname}: {e}")
+        # If resolution fails, assume it's unsafe or let the connection fail naturally,
+        # but to be safe we can return True to block it if strict mode is preferred.
+        # For now, we return False assuming standard DNS failure handling.
+        return False
+
+async def fetch_url_content(url: str, max_size: int = MAX_DOWNLOAD_SIZE) -> tuple[bytes, str]:
+    """
+    Fetch URL content with security checks (size limit, content type, private IP blocking).
+
+    Args:
+        url: The URL to fetch.
+        max_size: Maximum allowed size in bytes.
+
+    Returns:
+        tuple: (content_bytes, final_url)
+
+    Raises:
+        ValueError: If validation fails or download error occurs.
+    """
+    if not is_url_parsing_enabled():
+        raise ValueError("URL parsing feature is disabled")
+
+    # Initial validation
+    is_valid, error_msg = validate_url(url)
+    if not is_valid:
+        raise ValueError(f"Invalid URL: {error_msg}")
+
+    # Parse URL to check IP
+    parsed_url = urlparse(url)
+    if await is_private_ip(parsed_url.hostname):
+        raise ValueError("Access to private IP addresses is forbidden")
+
+    current_url = url
+    redirect_count = 0
+    max_redirects = 5
+    
+    # We handle redirects manually to check each target URL against whitelist/private IP
+    try:
+        async with httpx.AsyncClient(timeout=30.0, follow_redirects=False) as client:
+            while True:
+                logger.info(f"Fetching URL: {current_url}")
+                
+                # Request headers
+                headers = {
+                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+                }
+                
+                # Stream the response to check headers before downloading body
+                async with client.stream("GET", current_url, headers=headers) as response:
+                    # Handle Redirects
+                    if response.status_code in (301, 302, 303, 307, 308):
+                        if redirect_count >= max_redirects:
+                            raise ValueError("Too many redirects")
+                        
+                        redirect_count += 1
+                        location = response.headers.get("Location")
+                        if not location:
+                            raise ValueError("Redirect response missing Location header")
+                        
+                        # Handle relative redirects
+                        if location.startswith("/"):
+                            parsed_current = urlparse(current_url)
+                            current_url = f"{parsed_current.scheme}://{parsed_current.netloc}{location}"
+                        elif not location.startswith("http"):
+                            # Handle relative path without /? or other weird cases, or assume absolute
+                             parsed_current = urlparse(current_url)
+                             # simple join
+                             from urllib.parse import urljoin
+                             current_url = urljoin(current_url, location)
+                        else:
+                            current_url = location
+                        
+                        # Validate the new URL
+                        is_valid, error_msg = validate_url(current_url)
+                        if not is_valid:
+                            raise ValueError(f"Redirected to invalid URL: {error_msg}")
+                            
+                        parsed_new = urlparse(current_url)
+                        if await is_private_ip(parsed_new.hostname):
+                            raise ValueError("Redirected to private IP address")
+                            
+                        continue # Start new request
+                    
+                    response.raise_for_status()
+                    
+                    # Check Content-Type
+                    content_type = response.headers.get("Content-Type", "").lower()
+                    if not any(allowed in content_type for allowed in ALLOWED_CONTENT_TYPES):
+                        raise ValueError(f"Unsupported Content-Type: {content_type}. Only HTML is supported.")
+                    
+                    # Download content with size limit
+                    content = bytearray()
+                    async for chunk in response.aiter_bytes():
+                        content.extend(chunk)
+                        if len(content) > max_size:
+                            raise ValueError(f"Content size exceeds limit of {max_size} bytes")
+                    
+                    return bytes(content), current_url
+
+    except httpx.HTTPError as e:
+        logger.error(f"HTTP error fetching {url}: {e}")
+        raise ValueError(f"Failed to fetch URL: {e}")
+    except Exception as e:
+        logger.error(f"Error fetching {url}: {e}")
+        raise ValueError(f"Error fetching URL: {str(e)}")
--- a/src/knowledge/utils/url_validator.py
+++ b/src/knowledge/utils/url_validator.py
@ -0,0 +1,86 @@
+"""URL validation utilities for whitelist-based URL parsing."""
+import os
+from urllib.parse import urlparse
+
+# Environment variable name for URL whitelist
+YUXI_URL_WHITELIST_ENV = "YUXI_URL_WHITELIST"
+
+
+def _get_whitelist() -> list[str]:
+    """Get the URL whitelist from environment variables."""
+    whitelist_str = os.environ.get(YUXI_URL_WHITELIST_ENV, "")
+    if not whitelist_str:
+        return []
+    # Split by comma and clean up whitespace
+    return [item.strip() for item in whitelist_str.split(",") if item.strip()]
+
+
+def validate_url(url: str) -> tuple[bool, str]:
+    """
+    Validate if a URL is in the whitelist.
+
+    Args:
+        url: The URL to validate.
+
+    Returns:
+        A tuple of (is_valid, error_message).
+        If is_valid is True, error_message will be empty.
+    """
+    if not url:
+        return False, "URL cannot be empty"
+
+    # Parse the URL
+    try:
+        parsed = urlparse(url)
+    except Exception as e:
+        return False, f"Invalid URL format: {e}"
+
+    # Check if URL has a valid scheme
+    if not parsed.scheme:
+        return False, "URL must start with http:// or https://"
+
+    if parsed.scheme not in ("http", "https"):
+        return False, "URL must use HTTP or HTTPS protocol"
+
+    # Get the hostname
+    hostname = parsed.hostname
+    if not hostname:
+        return False, "Invalid URL: no hostname found"
+
+    # Get the whitelist
+    whitelist = _get_whitelist()
+
+    # If whitelist is empty, URL parsing is disabled
+    if not whitelist:
+        return False, "URL parsing feature is disabled"
+
+    # Check if hostname is in whitelist
+    for allowed in whitelist:
+        # Handle wildcard patterns like *.example.com
+        if allowed.startswith("*."):
+            domain = allowed[2:]
+            # Check if hostname ends with the domain (or matches exactly)
+            if hostname == domain or hostname.endswith(f".{domain}"):
+                return True, ""
+        else:
+            # Exact match or subdomain match
+            if hostname == allowed or hostname.endswith(f".{allowed}"):
+                return True, ""
+
+    return False, f"Domain '{hostname}' is not in the whitelist"
+
+
+def is_url_parsing_enabled() -> bool:
+    """Check if URL parsing is enabled (whitelist is configured)."""
+    whitelist = _get_whitelist()
+    return len(whitelist) > 0
+
+
+def get_whitelist_info() -> dict:
+    """Get information about the current whitelist configuration."""
+    whitelist = _get_whitelist()
+    return {
+        "enabled": len(whitelist) > 0,
+        "domains": whitelist,
+        "count": len(whitelist),
+    }
--- a/web/src/apis/knowledge_api.js
+++ b/web/src/apis/knowledge_api.js
@ -259,6 +259,19 @@ export const queryApi = {
 // =============================================================================

 export const fileApi = {
+  /**
+   * 抓取 URL 内容
+   * @param {string} url - 目标 URL
+   * @param {string} dbId - 知识库 ID
+   * @returns {Promise} - 抓取结果
+   */
+  fetchUrl: async (url, dbId = null) => {
+    return apiAdminPost('/api/knowledge/files/fetch-url', {
+      url,
+      db_id: dbId
+    })
+  },
+
  /**
   * 上传文件
   * @param {File} file - 文件对象
--- a/web/src/components/FileTable.vue
+++ b/web/src/components/FileTable.vue
@ -24,6 +24,13 @@
              >
                上传文件夹
              </a-menu-item>
+              <a-menu-item
+                key="upload-url"
+                @click="showAddFilesModal({ mode: 'url' })"
+                :icon="h(Link, { size: 16 })"
+              >
+                解析 URL
+              </a-menu-item>
            </a-menu>
          </template>
        </a-dropdown>
@ -342,6 +349,7 @@
                    @click="handleDownloadFile(record)"
                    :disabled="
                      lock ||
+                      record.file_type === 'url' ||
                      !['done', 'indexed', 'parsed', 'error_indexing'].includes(record.status)
                    "
                  >
@ -441,7 +449,8 @@ import {
  Search,
  Filter,
  ArrowUpDown,
-  ChevronDown
+  ChevronDown,
+  Link
 } from 'lucide-vue-next'

 const store = useDatabaseStore()
@ -831,7 +840,10 @@ const buildFileTree = (fileList) => {
    const normalizedName = file.filename.replace(/\\/g, '/')
    const parts = normalizedName.split('/')

-    if (parts.length === 1) {
+    // 检测是否是 URL（URL 不应该被解析为文件夹层级）
+    const isUrl = file.filename.startsWith('http://') || file.filename.startsWith('https://')
+
+    if (isUrl || parts.length === 1) {
      // Root item
      // Check if it's an explicit folder that should merge with an existing implicit one?
      if (item.is_folder) {
--- a/web/src/components/FileUploadModal.vue
+++ b/web/src/components/FileUploadModal.vue
@ -12,7 +12,11 @@
            type="primary"
            @click="chunkData"
            :loading="chunkLoading"
-            :disabled="fileList.length === 0"
+            :disabled="
+              uploadMode === 'url'
+                ? !urlList.some((i) => i.status === 'success')
+                : fileList.length === 0
+            "
          >
            添加到知识库
          </a-button>
@ -30,13 +34,23 @@
            class="custom-segmented"
          />
        </div>
+        <div class="auto-index-toggle">
+          <a-checkbox v-model:checked="autoIndex">上传后自动入库</a-checkbox>
+        </div>
      </div>

      <!-- 2. 配置面板 -->
-      <div class="settings-panel">
+      <div
+        class="settings-panel"
+        v-if="folderTreeData.length > 0 || uploadMode !== 'url' || autoIndex"
+      >
        <!-- 第一行：存储位置 + OCR 引擎 -->
-        <div class="setting-row two-cols">
-          <div class="col-item">
+        <div
+          class="setting-row"
+          v-if="folderTreeData.length > 0 || uploadMode !== 'url'"
+          :class="{ 'two-cols': uploadMode !== 'url' && folderTreeData.length > 0 }"
+        >
+          <div class="col-item" v-if="folderTreeData.length > 0">
            <div class="setting-label">存储位置</div>
            <div class="setting-content flex-row">
              <a-tree-select
@ -54,7 +68,7 @@
            </div>
            <p class="param-description">选择文件保存的目标文件夹</p>
          </div>
-          <div class="col-item">
+          <div class="col-item" v-if="uploadMode !== 'url'">
            <div class="setting-label">
              OCR 引擎
              <a-tooltip title="检查服务状态">
@ -89,13 +103,11 @@
          </div>
        </div>

-        <!-- 第三行：自动入库配置 -->
-        <div class="setting-row">
+        <!-- 第二行：自动入库配置 (仅在开启时显示) -->
+        <div class="setting-row" v-if="autoIndex">
          <div class="col-item">
-            <div class="setting-label">
-              <a-checkbox v-model:checked="autoIndex">上传后自动入库</a-checkbox>
-            </div>
-            <div class="setting-content" v-if="autoIndex">
+            <div class="setting-label">入库参数配置</div>
+            <div class="setting-content">
              <template v-if="!isGraphBased">
                <ChunkParamsConfig :temp-chunk-params="indexParams" :show-qa-split="true" />
              </template>
@ -117,7 +129,7 @@
      </div>

      <!-- 文件上传区域 -->
-      <div class="upload-area">
+      <div class="upload-area" v-if="uploadMode !== 'url'">
        <a-upload-dragger
          class="custom-dragger"
          v-model:fileList="fileList"
@ -139,6 +151,59 @@
        </a-upload-dragger>
      </div>

+      <!-- URL 输入区域 -->
+      <div class="url-area" v-if="uploadMode === 'url'">
+        <div class="url-input-wrapper">
+          <a-textarea
+            v-model:value="newUrl"
+            placeholder="输入 URL，一行一个&#10;https://site1.com&#10;https://site2.com"
+            :auto-size="{ minRows: 4, maxRows: 8 }"
+            class="url-input"
+            @keydown.enter.ctrl="handleFetchUrls"
+          />
+          <div class="url-actions">
+            <span class="url-hint">
+              支持批量粘贴，自动过滤空行。
+              <span class="warning-text">需配置白名单，详见文档说明</span>
+            </span>
+            <a-button
+              type="primary"
+              @click="handleFetchUrls"
+              class="add-url-btn"
+              :loading="fetchingUrls"
+              :disabled="!newUrl.trim()"
+            >
+              加载 URLs
+            </a-button>
+          </div>
+        </div>
+        <div class="url-list" v-if="urlList.length > 0">
+          <div v-for="(item, index) in urlList" :key="index" class="url-item">
+            <div class="url-icon-wrapper">
+              <Link v-if="item.status === 'success'" :size="14" class="url-icon success" />
+              <Info
+                v-else-if="item.status === 'error'"
+                :size="14"
+                class="url-icon error"
+                :title="item.error"
+              />
+              <RotateCw v-else :size="14" class="url-icon spinning" />
+            </div>
+            <div class="url-content">
+              <span class="url-text" :title="item.url">{{ item.url }}</span>
+              <span v-if="item.status === 'error'" class="url-error-msg">{{ item.error }}</span>
+            </div>
+            <a-button type="text" size="small" class="remove-url-btn" @click="removeUrl(index)">
+              <X :size="14" />
+            </a-button>
+          </div>
+        </div>
+        <div class="url-empty-tip" v-else>
+          <Info :size="16" />
+          <span>输入 URL 后点击加载，系统将自动抓取网页内容</span>
+        </div>
+      </div>
+
      <!-- 同名文件提示 -->
      <div v-if="sameNameFiles.length > 0" class="conflict-files-panel">
        <div class="panel-header">
@ -185,7 +250,17 @@ import { useDatabaseStore } from '@/stores/database'
 import { ocrApi } from '@/apis/system_api'
 import { fileApi, documentApi } from '@/apis/knowledge_api'
 import { CheckCircleFilled, ReloadOutlined } from '@ant-design/icons-vue'
-import { FileUp, FolderUp, RotateCw, CircleHelp, Info, Download, Trash2 } from 'lucide-vue-next'
+import {
+  FileUp,
+  FolderUp,
+  RotateCw,
+  CircleHelp,
+  Info,
+  Download,
+  Trash2,
+  Link,
+  X
+} from 'lucide-vue-next'
 import { h } from 'vue'
 import ChunkParamsConfig from '@/components/ChunkParamsConfig.vue'

@ -205,6 +280,10 @@ const props = defineProps({
  isFolderMode: {
    type: Boolean,
    default: false
+  },
+  mode: {
+    type: String,
+    default: 'file'
  }
 })

@ -238,7 +317,7 @@ watch(
    if (newVal) {
      selectedFolderId.value = props.currentFolderId
      isFolderUpload.value = props.isFolderMode
-      uploadMode.value = props.isFolderMode ? 'folder' : 'file'
+      uploadMode.value = props.mode || (props.isFolderMode ? 'folder' : 'file')
    }
  }
 )
@ -343,23 +422,125 @@ const uploadModeOptions = computed(() => [
      h(FolderUp, { size: 16, class: 'option-icon' }),
      h('span', { class: 'option-text' }, '上传文件夹')
    ])
+  },
+  {
+    value: 'url',
+    label: h('div', { class: 'segmented-option' }, [
+      h(Link, { size: 16, class: 'option-icon' }),
+      h('span', { class: 'option-text' }, '解析 URL')
+    ])
  }
 ])

 watch(uploadMode, (val) => {
  isFolderUpload.value = val === 'folder'
-  // 切换模式时清空已选文件，避免混淆
+  // 切换模式时清空已选内容，避免混淆
  fileList.value = []
  sameNameFiles.value = []
+  urlList.value = []
+  newUrl.value = ''
 })

 // 文件列表
 const fileList = ref([])

+// URL 列表
+// Item structure: { url: string, status: 'fetching'|'success'|'error', data: object|null, error: string }
+const urlList = ref([])
+const newUrl = ref('')
+const fetchingUrls = ref(false)
+
 // 同名文件列表（用于显示提示）
 const sameNameFiles = ref([])

-// URL相关功能已移除
+// URL 相关功能
+const isValidUrl = (string) => {
+  try {
+    const url = new URL(string)
+    return url.protocol === 'http:' || url.protocol === 'https:'
+  } catch (_) {
+    return false
+  }
+}
+
+const handleFetchUrls = async () => {
+  const text = newUrl.value
+  if (!text) return
+
+  const lines = text
+    .split(/[\r\n]+/)
+    .map((l) => l.trim())
+    .filter((l) => l)
+  if (lines.length === 0) return
+
+  // 1. 预处理：添加到列表
+  const newItems = []
+  for (const url of lines) {
+    if (!isValidUrl(url)) {
+      continue
+    }
+    if (urlList.value.some((u) => u.url === url)) continue
+
+    const item = { url, status: 'pending', data: null, error: '' }
+    urlList.value.push(item)
+    newItems.push(item)
+  }
+
+  if (newItems.length === 0) {
+    if (lines.length > 0) {
+      message.warning('没有检测到有效的新 URL')
+    }
+    return
+  }
+
+  newUrl.value = '' // 清空输入框
+  fetchingUrls.value = true
+
+  // 2. 并发处理
+  // 为避免过多并发请求，可以考虑使用 p-limit 或简单的分批，但此处直接并发
+  const processItem = async (item) => {
+    item.status = 'fetching'
+    try {
+      const res = await fileApi.fetchUrl(item.url, databaseId.value)
+      item.status = 'success'
+      item.data = res
+
+      // 处理同名文件冲突提示
+      if (res.has_same_name && res.same_name_files && res.same_name_files.length > 0) {
+        // 合并到现有的同名文件列表中，去重
+        const existingIds = new Set(sameNameFiles.value.map((f) => f.file_id))
+        const newConflicts = res.same_name_files.filter((f) => !existingIds.has(f.file_id))
+        sameNameFiles.value.push(...newConflicts)
+      }
+    } catch (error) {
+      console.error('Failed to fetch URL:', error)
+      item.status = 'error'
+
+      // 特别处理内容重复 (409)
+      const detail = error.response?.data?.detail || error.message || ''
+      if (detail.includes('same content') || detail.includes('相同内容')) {
+        item.error = '内容已存在于知识库中'
+      } else {
+        item.error = detail || '加载失败'
+      }
+    }
+  }
+
+  await Promise.all(newItems.map(processItem))
+  fetchingUrls.value = false
+}
+
+const removeUrl = (index) => {
+  urlList.value.splice(index, 1)
+}
+
+const handleUrlKeydown = (e) => {
+  // Ctrl + Enter 提交
+  if (e.key === 'Enter' && e.ctrlKey) {
+    e.preventDefault()
+    handleFetchUrls()
+  }
+}

 // OCR服务健康状态
 const ocrHealthStatus = ref({
@ -790,11 +971,68 @@ const chunkData = async () => {
    return
  }

-  // 验证OCR服务可用性
-  if (!validateOcrService()) {
+  // 验证OCR服务可用性（非 URL 模式下）
+  if (uploadMode.value !== 'url' && !validateOcrService()) {
    return
  }

+  // URL 模式处理
+  if (uploadMode.value === 'url') {
+    // 过滤出成功的项
+    const successfulItems = urlList.value.filter((item) => item.status === 'success' && item.data)
+    if (successfulItems.length === 0) {
+      message.error('请添加并等待至少一个 URL 解析成功')
+      return
+    }
+
+    try {
+      store.state.chunkLoading = true
+      const params = { ...chunkParams.value }
+      if (autoIndex.value) {
+        params.auto_index = true
+        Object.assign(params, indexParams.value)
+      }
+
+      // 构造 _preprocessed_map 和 items (minio urls)
+      const items = []
+      const preprocessedMap = {}
+      for (const item of successfulItems) {
+        // item.data = { file_path: "http://minio...", content_hash: "...", filename: "...", ... }
+        // 注意：fetch-url 返回的 file_path 其实是 MinIO URL
+        // 我们需要传递 MinIO URL 给 addDocuments
+        const minioUrl = item.data.file_path
+        items.push(minioUrl)
+        preprocessedMap[minioUrl] = {
+          path: minioUrl,
+          content_hash: item.data.content_hash,
+          filename: item.data.filename,
+          file_size: item.data.size
+        }
+      }
+      params._preprocessed_map = preprocessedMap
+
+      // 调用 addFiles (file mode)
+      await store.addFiles({
+        items: items,
+        contentType: 'file', // 重要：这里改为 file，因为我们已经转成了 minio 上的文件
+        params,
+        parentId: selectedFolderId.value
+      })
+
+      emit('success')
+      handleCancel()
+      urlList.value = []
+      newUrl.value = ''
+    } catch (error) {
+      console.error('URL 提交失败:', error)
+      message.error('URL 提交失败: ' + (error.message || '未知错误'))
+    } finally {
+      store.state.chunkLoading = false
+    }
+    return
+  }
+
+  // 文件模式处理
  const imageExtensions = ['.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif']

  // 提取已上传的文件信息
@ -880,12 +1118,25 @@ const chunkData = async () => {
  align-items: center;
 }

+.auto-index-toggle {
+  display: flex;
+  align-items: center;
+  padding-right: 4px;
+
+  :deep(.ant-checkbox-wrapper) {
+    font-size: 13px;
+    color: var(--gray-600);
+    font-weight: 500;
+  }
+}
+
 .help-link-btn {
  color: var(--gray-600);
  font-size: 13px;
  display: flex;
  align-items: center;
  gap: 6px;
+  padding: 0;

  &:hover {
    color: var(--main-color);
@ -1127,6 +1378,135 @@ const chunkData = async () => {
  border-radius: 4px;
 }

+/* URL Area */
+.url-area {
+  flex: 1;
+  display: flex;
+  flex-direction: column;
+  gap: 12px;
+}
+
+.url-input-wrapper {
+  width: 100%;
+}
+
+.url-actions {
+  display: flex;
+  align-items: center;
+  justify-content: flex-end;
+  gap: 12px;
+  margin-top: 8px;
+}
+
+.url-hint {
+  font-size: 12px;
+  color: var(--gray-500);
+
+  .warning-text {
+    color: var(--color-warning-500);
+    margin-left: 4px;
+  }
+}
+
+.url-input {
+  width: 100%;
+  padding: 10px;
+}
+
+.add-url-btn {
+  margin-left: 8px;
+}
+
+.url-list {
+  display: flex;
+  flex-direction: column;
+  gap: 8px;
+  max-height: 300px;
+  overflow-y: auto;
+}
+
+.url-item {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  padding: 8px 12px;
+  background: var(--gray-50);
+  border: 1px solid var(--gray-200);
+  border-radius: 6px;
+  transition: all 0.2s;
+
+  &:hover {
+    background: var(--gray-100);
+    border-color: var(--main-300);
+  }
+}
+
+.url-icon-wrapper {
+  display: flex;
+  align-items: center;
+  flex-shrink: 0;
+}
+
+.url-icon {
+  color: var(--main-500);
+
+  &.success {
+    color: var(--color-success-500);
+  }
+
+  &.error {
+    color: var(--color-error-500);
+    cursor: help;
+  }
+
+  &.spinning {
+    animation: spin 1s linear infinite;
+    color: var(--main-500);
+  }
+}
+
+.url-content {
+  flex: 1;
+  display: flex;
+  flex-direction: column;
+  overflow: hidden;
+}
+
+.url-text {
+  font-size: 13px;
+  color: var(--gray-700);
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+}
+
+.url-error-msg {
+  font-size: 11px;
+  color: var(--color-error-500);
+  margin-top: 2px;
+}
+
+.remove-url-btn {
+  color: var(--gray-400);
+  flex-shrink: 0;
+
+  &:hover {
+    color: var(--color-error-500);
+  }
+}
+
+.url-empty-tip {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  padding: 16px;
+  background: var(--gray-50);
+  border: 1px dashed var(--gray-300);
+  border-radius: 8px;
+  color: var(--gray-500);
+  font-size: 13px;
+}
+
 /* Conflict Files Panel */
 .conflict-files-panel {
  border: 1px solid var(--gray-200);
--- a/web/src/utils/file_utils.js
+++ b/web/src/utils/file_utils.js
@ -7,7 +7,8 @@ import {
  FileExcelFilled,
  FileImageFilled,
  FileUnknownFilled,
-  FilePptFilled
+  FilePptFilled,
+  LinkOutlined
 } from '@ant-design/icons-vue'
 import { formatRelative, parseToShanghai } from '@/utils/time'

@ -15,6 +16,11 @@ import { formatRelative, parseToShanghai } from '@/utils/time'
 export const getFileIcon = (filename) => {
  if (!filename) return FileUnknownFilled

+  // Check if it's a URL
+  if (filename.startsWith('http://') || filename.startsWith('https://')) {
+    return LinkOutlined
+  }
+
  const extension = filename.toLowerCase().split('.').pop()

  const iconMap = {
@ -64,6 +70,11 @@ export const getFileIcon = (filename) => {
 export const getFileIconColor = (filename) => {
  if (!filename) return '#8c8c8c'

+  // Check if it's a URL
+  if (filename.startsWith('http://') || filename.startsWith('https://')) {
+    return '#1890ff' // Blue for links
+  }
+
  const extension = filename.toLowerCase().split('.').pop()

  const colorMap = {
--- a/web/src/views/DataBaseInfoView.vue
+++ b/web/src/views/DataBaseInfoView.vue
@ -14,6 +14,7 @@
      :folder-tree="folderTree"
      :current-folder-id="currentFolderId"
      :is-folder-mode="isFolderUploadMode"
+      :mode="addFilesMode"
      @success="onFileUploadSuccess"
    />

@ -314,14 +315,16 @@ const openSearchConfigModal = () => {
 const addFilesModalVisible = ref(false)
 const currentFolderId = ref(null)
 const isFolderUploadMode = ref(false)
+const addFilesMode = ref('file')

 // 标记是否是初次加载
 const isInitialLoad = ref(true)

 // 显示添加文件弹窗
 const showAddFilesModal = (options = {}) => {
-  const { isFolder = false } = options
+  const { isFolder = false, mode = 'file' } = options
  isFolderUploadMode.value = isFolder
+  addFilesMode.value = mode
  addFilesModalVisible.value = true
  currentFolderId.value = null // 重置
 }