diff --git a/server/routers/knowledge_router.py b/server/routers/knowledge_router.py index 2cbe2b5a..65057788 100644 --- a/server/routers/knowledge_router.py +++ b/server/routers/knowledge_router.py @@ -210,6 +210,10 @@ async def add_documents( content_type = params.get("content_type", "file") + # 禁止 URL 解析与入库 + if content_type == "url": + raise HTTPException(status_code=400, detail="URL 文档上传与解析已禁用") + # 安全检查:验证文件路径 if content_type == "file": from src.knowledge.utils.kb_utils import validate_file_path diff --git a/src/knowledge/implementations/chroma.py b/src/knowledge/implementations/chroma.py index 8f9078ee..01bbab90 100644 --- a/src/knowledge/implementations/chroma.py +++ b/src/knowledge/implementations/chroma.py @@ -9,7 +9,7 @@ from chromadb.config import Settings from chromadb.utils.embedding_functions import OpenAIEmbeddingFunction from src.knowledge.base import KnowledgeBase -from src.knowledge.indexing import process_file_to_markdown, process_url_to_markdown +from src.knowledge.indexing import process_file_to_markdown from src.knowledge.utils.kb_utils import ( get_embedding_config, prepare_item_metadata, @@ -204,10 +204,9 @@ class ChromaKB(KnowledgeBase): params["db_id"] = db_id # 根据内容类型处理内容 - if content_type == "file": - markdown_content = await process_file_to_markdown(item, params=params) - else: # URL - markdown_content = await process_url_to_markdown(item, params=params) + if content_type != "file": + raise ValueError("URL 内容解析已禁用") + markdown_content = await process_file_to_markdown(item, params=params) # 分割文本成块 chunks = self._split_text_into_chunks(markdown_content, file_id, filename, params) @@ -296,10 +295,9 @@ class ChromaKB(KnowledgeBase): self._save_metadata() # 重新解析文件为 markdown - if content_type == "file": - markdown_content = await process_file_to_markdown(file_path, params=params) - else: - markdown_content = await process_url_to_markdown(file_path, params=params) + if content_type != "file": + raise ValueError("URL 内容解析已禁用") + markdown_content = await process_file_to_markdown(file_path, params=params) # 先删除现有的 ChromaDB 数据(仅删除chunks,保留元数据) await self.delete_file_chunks_only(db_id, file_id) diff --git a/src/knowledge/implementations/lightrag.py b/src/knowledge/implementations/lightrag.py index fb4d8e09..475a56bc 100644 --- a/src/knowledge/implementations/lightrag.py +++ b/src/knowledge/implementations/lightrag.py @@ -10,7 +10,7 @@ from pymilvus import connections, utility from src import config from src.knowledge.base import KnowledgeBase -from src.knowledge.indexing import process_file_to_markdown, process_url_to_markdown +from src.knowledge.indexing import process_file_to_markdown from src.knowledge.utils.kb_utils import get_embedding_config, prepare_item_metadata from src.utils import hashstr, logger from src.utils.datetime_utils import shanghai_now @@ -243,12 +243,11 @@ class LightRagKB(KnowledgeBase): params["db_id"] = db_id # 根据内容类型处理内容 - if content_type == "file": - markdown_content = await process_file_to_markdown(item, params=params) - markdown_content_lines = markdown_content[:100].replace("\n", " ") - logger.info(f"Markdown content: {markdown_content_lines}...") - else: # URL - markdown_content = await process_url_to_markdown(item, params=params) + if content_type != "file": + raise ValueError("URL 内容解析已禁用") + markdown_content = await process_file_to_markdown(item, params=params) + markdown_content_lines = markdown_content[:100].replace("\n", " ") + logger.info(f"Markdown content: {markdown_content_lines}...") # 使用 LightRAG 插入内容 await rag.ainsert(input=markdown_content, ids=file_id, file_paths=item_path) @@ -313,12 +312,11 @@ class LightRagKB(KnowledgeBase): self._save_metadata() # 重新解析文件为 markdown - if content_type == "file": - markdown_content = await process_file_to_markdown(file_path, params=params) - markdown_content_lines = markdown_content[:100].replace("\n", " ") - logger.info(f"Markdown content: {markdown_content_lines}...") - else: - markdown_content = await process_url_to_markdown(file_path, params=params) + if content_type != "file": + raise ValueError("URL 内容解析已禁用") + markdown_content = await process_file_to_markdown(file_path, params=params) + markdown_content_lines = markdown_content[:100].replace("\n", " ") + logger.info(f"Markdown content: {markdown_content_lines}...") # 先删除现有的 LightRAG 数据(仅删除chunks,保留元数据) await self.delete_file_chunks_only(db_id, file_id) diff --git a/src/knowledge/implementations/milvus.py b/src/knowledge/implementations/milvus.py index 549f73ce..3f626eb5 100644 --- a/src/knowledge/implementations/milvus.py +++ b/src/knowledge/implementations/milvus.py @@ -8,7 +8,7 @@ from typing import Any from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connections, db, utility from src.knowledge.base import KnowledgeBase -from src.knowledge.indexing import process_file_to_markdown, process_url_to_markdown +from src.knowledge.indexing import process_file_to_markdown from src.knowledge.utils.kb_utils import ( get_embedding_config, prepare_item_metadata, @@ -247,10 +247,9 @@ class MilvusKB(KnowledgeBase): params = {} params["db_id"] = db_id - if content_type == "file": - markdown_content = await process_file_to_markdown(item, params=params) - else: - markdown_content = await process_url_to_markdown(item, params=params) + if content_type != "file": + raise ValueError("URL 内容解析已禁用") + markdown_content = await process_file_to_markdown(item, params=params) chunks = self._split_text_into_chunks(markdown_content, file_id, filename, params) logger.info(f"Split {filename} into {len(chunks)} chunks") @@ -342,10 +341,9 @@ class MilvusKB(KnowledgeBase): self._save_metadata() # 重新解析文件为 markdown - if content_type == "file": - markdown_content = await process_file_to_markdown(file_path, params=params) - else: - markdown_content = await process_url_to_markdown(file_path, params=params) + if content_type != "file": + raise ValueError("URL 内容解析已禁用") + markdown_content = await process_file_to_markdown(file_path, params=params) # 先删除现有的 Milvus 数据(仅删除chunks,保留元数据) await self.delete_file_chunks_only(db_id, file_id) diff --git a/src/knowledge/indexing.py b/src/knowledge/indexing.py index 7e792d22..30f6e936 100644 --- a/src/knowledge/indexing.py +++ b/src/knowledge/indexing.py @@ -551,24 +551,4 @@ def _replace_image_links(markdown_content: str, images: list[dict]) -> str: async def process_url_to_markdown(url: str, params: dict | None = None) -> str: - """ - 将URL转换为markdown格式 - - Args: - url: URL地址 - params: 处理参数 - - Returns: - markdown格式内容 - """ - import requests - from bs4 import BeautifulSoup - - try: - response = requests.get(url, timeout=30) - soup = BeautifulSoup(response.content, "html.parser") - text_content = soup.get_text() - return f"# {url}\n\n{text_content}" - except Exception as e: - logger.error(f"Failed to process URL {url}: {e}") - return f"# {url}\n\nFailed to process URL: {e}" + raise NotImplementedError("URL 解析功能已禁用") diff --git a/src/knowledge/utils/kb_utils.py b/src/knowledge/utils/kb_utils.py index 5ea50fd9..322d9362 100644 --- a/src/knowledge/utils/kb_utils.py +++ b/src/knowledge/utils/kb_utils.py @@ -150,12 +150,8 @@ def prepare_item_metadata(item: str, content_type: str, db_id: str, params: dict content_hash = calculate_content_hash(file_path) except Exception as exc: # noqa: BLE001 logger.warning(f"Failed to calculate content hash for {file_path}: {exc}") - else: # URL - file_id = f"url_{hashstr(item + str(time.time()), 6)}" - file_type = "url" - filename = f"webpage_{hashstr(item, 6)}.md" - item_path = item - content_hash = None + else: + raise ValueError("URL 元数据生成已禁用") metadata = { "database_id": db_id, diff --git a/web/src/components/FileUploadModal.vue b/web/src/components/FileUploadModal.vue index 582fa366..3e2a7253 100644 --- a/web/src/components/FileUploadModal.vue +++ b/web/src/components/FileUploadModal.vue @@ -12,7 +12,7 @@ type="primary" @click="chunkData" :loading="chunkLoading" - :disabled="(uploadMode === 'file' && fileList.length === 0) || (uploadMode === 'url' && !urlList.trim())" + :disabled="fileList.length === 0" > 添加到知识库 @@ -26,6 +26,7 @@ :options="uploadModeOptions" size="large" class="source-segmented" + :disabled="true" />
@@ -101,24 +102,7 @@
- - - -
- - - - - -

- 支持添加网页内容,系统会自动抓取网页文本并进行分块。请确保URL格式正确且可以公开访问。 -

-
+ @@ -139,7 +123,7 @@