diff --git a/pyproject.toml b/pyproject.toml index 7651724a..e1c8c006 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "rapidocr-onnxruntime>=1.4.4", "sentencepiece>=0.2.0", "tavily-python>=0.7.0", + "unstructured>=0.17.2", "uvicorn[standard]>=0.34.2", "zhipuai>=2.1.5.20250421", ] diff --git a/server/routers/data_router.py b/server/routers/data_router.py index 3b553408..55d07121 100644 --- a/server/routers/data_router.py +++ b/server/routers/data_router.py @@ -58,6 +58,12 @@ async def file_to_chunk(files: List[str] = Body(...), params: dict = Body(...), result = knowledge_base.file_to_chunk(files, params=params) return result +@data.post("/url-to-chunk") +async def url_to_chunk(urls: List[str] = Body(...), params: dict = Body(...), current_user: User = Depends(get_admin_user)): + logger.debug(f"Url to chunk: {urls}") + result = knowledge_base.url_to_chunk(urls, params=params) + return result + @data.post("/add-by-file") async def create_document_by_file(db_id: str = Body(...), files: List[str] = Body(...), current_user: User = Depends(get_admin_user)): logger.debug(f"Add document in {db_id} by file: {files}") diff --git a/src/core/graphbase.py b/src/core/graphbase.py index 7f94e852..3b446fd5 100644 --- a/src/core/graphbase.py +++ b/src/core/graphbase.py @@ -258,6 +258,7 @@ class GraphDatabase: tx.run(query) def query_node(self, entity_name, threshold=0.9, kgdb_name='neo4j', hops=2, max_entities=5, **kwargs): + """知识图谱查询节点的入口:""" # TODO 添加判断节点数量为 0 停止检索 # 判断是否启动 if not self.is_running(): diff --git a/src/core/knowledgebase.py b/src/core/knowledgebase.py index df338702..78587aae 100644 --- a/src/core/knowledgebase.py +++ b/src/core/knowledgebase.py @@ -208,9 +208,75 @@ class KnowledgeBase: return file_infos - def url_to_chunk(self, url, params=None): - """将url转换为分块，读取url的内容，并转换为分块""" - raise NotImplementedError("Not implemented") + def url_to_chunk(self, urls, params=None): + """将url转换为分块，读取url的内容，并转换为分块 + + Args: + urls: list of urls + params: params for chunking + + Returns: + dict: 包含分块信息的字典 + """ + try: + from langchain_community.document_loaders import UnstructuredURLLoader + except ImportError: + raise ImportError("请安装 langchain_community 和 unstructured 包：pip install langchain-community unstructured") + + file_infos = {} + + # 使用UnstructuredURLLoader加载URL内容 + loader = UnstructuredURLLoader(urls=urls, continue_on_failure=True) + + for url_idx, url in enumerate(urls): + file_id = "url_" + hashstr(url + str(time.time())) + + try: + # 加载单个URL内容 + single_loader = UnstructuredURLLoader(urls=[url], continue_on_failure=False) + documents = single_loader.load() + + # 将文档内容合并 + text_content = "\n\n".join([doc.page_content for doc in documents]) + + # 对内容进行分块 + nodes = chunk(text_content, params=params) + + # 从URL中提取域名作为文件名 + from urllib.parse import urlparse, unquote + parsed_url = urlparse(unquote(url)) + domain = parsed_url.netloc + path = parsed_url.path + filename = f"{domain}{path}" + if filename.endswith('/'): + filename = filename[:-1] + if len(filename) > 100: + filename = filename[:97] + "..." + filename = filename.replace('/', '_') + + file_infos[file_id] = { + "file_id": file_id, + "filename": filename, + "path": url, + "type": "url", + "status": "waiting", + "created_at": time.time(), + "nodes": [node.dict() for node in nodes] + } + except Exception as e: + logger.error(f"处理URL {url} 时出错: {e}") + file_infos[file_id] = { + "file_id": file_id, + "filename": url[:100] + "..." if len(url) > 100 else url, + "path": url, + "type": "url", + "status": "failed", + "created_at": time.time(), + "error": str(e), + "nodes": [] + } + + return file_infos def add_chunks(self, db_id, file_chunks): """添加分块""" diff --git a/web/src/apis/admin_api.js b/web/src/apis/admin_api.js index 10265634..b0a9596b 100644 --- a/web/src/apis/admin_api.js +++ b/web/src/apis/admin_api.js @@ -180,7 +180,17 @@ export const knowledgeBaseApi = { getDocumentDetail: async (dbId, fileId) => { checkAdminPermission() return apiGet(`/api/data/document?db_id=${dbId}&file_id=${fileId}`, {}, true) - } + }, + + /** + * 将URL转换为分块 + * @param {Object} data - 分块参数 + * @returns {Promise} - 分块结果 + */ + urlToChunk: async (data) => { + checkAdminPermission() + return apiPost('/api/data/url-to-chunk', data, {}, true) + }, } // 图数据库管理API diff --git a/web/src/apis/base.js b/web/src/apis/base.js index 9e6a7eee..8ad65fb4 100644 --- a/web/src/apis/base.js +++ b/web/src/apis/base.js @@ -40,7 +40,7 @@ export async function apiRequest(url, options = {}, requiresAuth = false) { // 处理API返回的错误 if (!response.ok) { // 尝试解析错误信息 - let errorMessage = `请求失败: ${response.status}` + let errorMessage = `请求失败: ${response.status}, ${response.statusText}` let errorData = null try { diff --git a/web/src/components/LoadingComponent.vue b/web/src/components/LoadingComponent.vue new file mode 100644 index 00000000..617de90c --- /dev/null +++ b/web/src/components/LoadingComponent.vue @@ -0,0 +1,64 @@ + + + + + \ No newline at end of file diff --git a/web/src/views/DataBaseInfoView.vue b/web/src/views/DataBaseInfoView.vue index bcf58d4d..5fbf04d4 100644 --- a/web/src/views/DataBaseInfoView.vue +++ b/web/src/views/DataBaseInfoView.vue @@ -107,7 +107,19 @@
-
+
+ + + 上传文件 + + + 输入网址 + + +
+ + +
+ + +
+ + + + + +

+ 支持添加网页内容，系统会自动抓取网页文本并进行分块。请确保URL格式正确且可以公开访问。 +

+
+
生成分块 @@ -299,6 +329,8 @@ import { CloudUploadOutlined, SearchOutlined, LoadingOutlined, + FileOutlined, + LinkOutlined, } from '@ant-design/icons-vue' @@ -623,6 +655,39 @@ const chunkFiles = () => { }) } +// 分块预览 +const chunkUrls = () => { + // 分割并过滤URL列表 + const urls = urlList.value.split('\n') + .map(url => url.trim()) + .filter(url => url.length > 0 && (url.startsWith('http://') || url.startsWith('https://'))); + + if (urls.length === 0) { + message.error('请输入有效的网页链接（必须以http://或https://开头）'); + return; + } + + state.loading = true; + + // 调用url-to-chunk接口获取分块信息 + knowledgeBaseApi.urlToChunk({ + urls: urls, + params: chunkParams.value + }) + .then(data => { + console.log('URL分块信息:', data); + chunkResults.value = Object.values(data); + activeFileKeys.value = chunkResults.value.length > 0 ? [0] : []; // 默认展开第一个 + }) + .catch(error => { + console.error(error); + message.error(error.message || '处理URL失败'); + }) + .finally(() => { + state.loading = false; + }); +}; + // 添加到数据库 const addToDatabase = () => { if (chunkResults.value.length === 0) { @@ -723,7 +788,16 @@ onUnmounted(() => { } }) +const uploadMode = ref('file'); +const urlList = ref(''); +const chunkData = () => { + if (uploadMode.value === 'file') { + chunkFiles(); + } else if (uploadMode.value === 'url') { + chunkUrls(); + } +} @@ -1181,6 +1255,29 @@ onUnmounted(() => { } } } + +.url-input { + margin-bottom: 20px; +} + +.url-input .ant-textarea { + border-color: var(--main-light-3); + background-color: #fff; + font-family: monospace; + resize: vertical; +} + +.url-input .ant-textarea:hover, +.url-input .ant-textarea:focus { + border-color: var(--main-color); +} + +.url-hint { + font-size: 13px; + color: var(--gray-600); + margin-top: 5px; + line-height: 1.5; +}