支持使用URL添加到知识库；

pyproject.toml

@@ -30,6 +30,7 @@ dependencies = [
     "rapidocr-onnxruntime>=1.4.4",
     "sentencepiece>=0.2.0",
     "tavily-python>=0.7.0",
+    "unstructured>=0.17.2",
     "uvicorn[standard]>=0.34.2",
     "zhipuai>=2.1.5.20250421",
 ]

server/routers/data_router.py

@@ -58,6 +58,12 @@ async def file_to_chunk(files: List[str] = Body(...), params: dict = Body(...),
     result = knowledge_base.file_to_chunk(files, params=params)
     return result
 
+@data.post("/url-to-chunk")
+async def url_to_chunk(urls: List[str] = Body(...), params: dict = Body(...), current_user: User = Depends(get_admin_user)):
+    logger.debug(f"Url to chunk: {urls}")
+    result = knowledge_base.url_to_chunk(urls, params=params)
+    return result
+
 @data.post("/add-by-file")
 async def create_document_by_file(db_id: str = Body(...), files: List[str] = Body(...), current_user: User = Depends(get_admin_user)):
     logger.debug(f"Add document in {db_id} by file: {files}")

src/core/graphbase.py

@@ -258,6 +258,7 @@ class GraphDatabase:
         tx.run(query)
 
     def query_node(self, entity_name, threshold=0.9, kgdb_name='neo4j', hops=2, max_entities=5, **kwargs):
+        """知识图谱查询节点的入口:"""
         # TODO 添加判断节点数量为 0 停止检索
         # 判断是否启动
         if not self.is_running():

src/core/knowledgebase.py

@@ -208,9 +208,75 @@ class KnowledgeBase:
 
         return file_infos
 
-    def url_to_chunk(self, url, params=None):
+    def url_to_chunk(self, urls, params=None):
-        """将url转换为分块，读取url的内容，并转换为分块"""
+        """将url转换为分块，读取url的内容，并转换为分块
-        raise NotImplementedError("Not implemented")
+
+        Args:
+            urls: list of urls
+            params: params for chunking
+
+        Returns:
+            dict: 包含分块信息的字典
+        """
+        try:
+            from langchain_community.document_loaders import UnstructuredURLLoader
+        except ImportError:
+            raise ImportError("请安装 langchain_community 和 unstructured 包：pip install langchain-community unstructured")
+
+        file_infos = {}
+
+        # 使用UnstructuredURLLoader加载URL内容
+        loader = UnstructuredURLLoader(urls=urls, continue_on_failure=True)
+
+        for url_idx, url in enumerate(urls):
+            file_id = "url_" + hashstr(url + str(time.time()))
+
+            try:
+                # 加载单个URL内容
+                single_loader = UnstructuredURLLoader(urls=[url], continue_on_failure=False)
+                documents = single_loader.load()
+
+                # 将文档内容合并
+                text_content = "\n\n".join([doc.page_content for doc in documents])
+
+                # 对内容进行分块
+                nodes = chunk(text_content, params=params)
+
+                # 从URL中提取域名作为文件名
+                from urllib.parse import urlparse, unquote
+                parsed_url = urlparse(unquote(url))
+                domain = parsed_url.netloc
+                path = parsed_url.path
+                filename = f"{domain}{path}"
+                if filename.endswith('/'):
+                    filename = filename[:-1]
+                if len(filename) > 100:
+                    filename = filename[:97] + "..."
+                filename = filename.replace('/', '_')
+
+                file_infos[file_id] = {
+                    "file_id": file_id,
+                    "filename": filename,
+                    "path": url,
+                    "type": "url",
+                    "status": "waiting",
+                    "created_at": time.time(),
+                    "nodes": [node.dict() for node in nodes]
+                }
+            except Exception as e:
+                logger.error(f"处理URL {url} 时出错: {e}")
+                file_infos[file_id] = {
+                    "file_id": file_id,
+                    "filename": url[:100] + "..." if len(url) > 100 else url,
+                    "path": url,
+                    "type": "url",
+                    "status": "failed",
+                    "created_at": time.time(),
+                    "error": str(e),
+                    "nodes": []
+                }
+
+        return file_infos
 
     def add_chunks(self, db_id, file_chunks):
         """添加分块"""

web/src/apis/admin_api.js

@@ -180,7 +180,17 @@ export const knowledgeBaseApi = {
   getDocumentDetail: async (dbId, fileId) => {
     checkAdminPermission()
     return apiGet(`/api/data/document?db_id=${dbId}&file_id=${fileId}`, {}, true)
-  }
+  },
+
+  /**
+   * 将URL转换为分块
+   * @param {Object} data - 分块参数
+   * @returns {Promise} - 分块结果
+   */
+  urlToChunk: async (data) => {
+    checkAdminPermission()
+    return apiPost('/api/data/url-to-chunk', data, {}, true)
+  },
 }
 
 // 图数据库管理API

web/src/apis/base.js

@@ -40,7 +40,7 @@ export async function apiRequest(url, options = {}, requiresAuth = false) {
     // 处理API返回的错误
     if (!response.ok) {
       // 尝试解析错误信息
-      let errorMessage = `请求失败: ${response.status}`
+      let errorMessage = `请求失败: ${response.status}, ${response.statusText}`
       let errorData = null
 
       try {

web/src/components/LoadingComponent.vue

@@ -0,0 +1,64 @@
+<template>
+  <div class="loading-container" v-if="visible">
+    <div class="loading-content">
+      <div class="loading-spinner"></div>
+      <div class="loading-text">{{ text }}</div>
+    </div>
+  </div>
+</template>
+
+<script setup>
+import { defineProps } from 'vue';
+
+const props = defineProps({
+  visible: {
+    type: Boolean,
+    default: false
+  },
+  text: {
+    type: String,
+    default: '加载中...'
+  }
+});
+</script>
+
+<style scoped>
+.loading-container {
+  position: fixed;
+  top: 0;
+  left: 0;
+  right: 0;
+  bottom: 0;
+  background-color: rgba(255, 255, 255, 0.8);
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  z-index: 9999;
+}
+
+.loading-content {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+}
+
+.loading-spinner {
+  width: 50px;
+  height: 50px;
+  border: 5px solid #f3f3f3;
+  border-top: 5px solid var(--main-color);
+  border-radius: 50%;
+  animation: spin 1s linear infinite;
+}
+
+.loading-text {
+  margin-top: 16px;
+  font-size: 16px;
+  color: var(--gray-900);
+}
+
+@keyframes spin {
+  0% { transform: rotate(0deg); }
+  100% { transform: rotate(360deg); }
+}
+</style> 

web/src/views/DataBaseInfoView.vue

@@ -107,7 +107,19 @@
               </div>
             </div>
             <div class="upload-main">
-              <div class="upload">
+              <div class="source-selector">
+                <a-radio-group v-model:value="uploadMode" button-style="solid" style="margin-bottom: 16px;">
+                  <a-radio-button value="file">
+                    <FileOutlined /> 上传文件
+                  </a-radio-button>
+                  <a-radio-button value="url">
+                    <LinkOutlined /> 输入网址
+                  </a-radio-button>
+                </a-radio-group>
+              </div>
+
+              <!-- 文件上传区域 -->
+              <div class="upload" v-if="uploadMode === 'file'">
                 <a-upload-dragger
                   class="upload-dragger"
                   v-model:fileList="fileList"
@@ -124,12 +136,30 @@
                   </p>
                 </a-upload-dragger>
               </div>
+
+              <!-- URL 输入区域 -->
+              <div class="url-input" v-else>
+                <a-form layout="vertical">
+                  <a-form-item label="网页链接 (每行一个URL)">
+                    <a-textarea
+                      v-model:value="urlList"
+                      placeholder="请输入网页链接，每行一个"
+                      :rows="6"
+                      :disabled="state.loading"
+                    />
+                  </a-form-item>
+                </a-form>
+                <p class="url-hint">
+                  支持添加网页内容，系统会自动抓取网页文本并进行分块。请确保URL格式正确且可以公开访问。
+                </p>
+              </div>
+
               <div class="actions">
                 <a-button
                   type="primary"
-                  @click="chunkFiles"
+                  @click="chunkData"
                   :loading="state.loading"
-                  :disabled="fileList.length === 0"
+                  :disabled="(uploadMode === 'file' && fileList.length === 0) || (uploadMode === 'url' && !urlList.trim())"
                   style="margin: 0px 20px 20px 0;"
                 >
                   生成分块
@@ -299,6 +329,8 @@ import {
   CloudUploadOutlined,
   SearchOutlined,
   LoadingOutlined,
+  FileOutlined,
+  LinkOutlined,
 } from '@ant-design/icons-vue'
 
 
@@ -623,6 +655,39 @@ const chunkFiles = () => {
   })
 }
 
+// 分块预览
+const chunkUrls = () => {
+  // 分割并过滤URL列表
+  const urls = urlList.value.split('\n')
+    .map(url => url.trim())
+    .filter(url => url.length > 0 && (url.startsWith('http://') || url.startsWith('https://')));
+
+  if (urls.length === 0) {
+    message.error('请输入有效的网页链接（必须以http://或https://开头）');
+    return;
+  }
+
+  state.loading = true;
+
+  // 调用url-to-chunk接口获取分块信息
+  knowledgeBaseApi.urlToChunk({
+    urls: urls,
+    params: chunkParams.value
+  })
+  .then(data => {
+    console.log('URL分块信息:', data);
+    chunkResults.value = Object.values(data);
+    activeFileKeys.value = chunkResults.value.length > 0 ? [0] : []; // 默认展开第一个
+  })
+  .catch(error => {
+    console.error(error);
+    message.error(error.message || '处理URL失败');
+  })
+  .finally(() => {
+    state.loading = false;
+  });
+};
+
 // 添加到数据库
 const addToDatabase = () => {
   if (chunkResults.value.length === 0) {
@@ -723,7 +788,16 @@ onUnmounted(() => {
   }
 })
 
+const uploadMode = ref('file');
+const urlList = ref('');
 
+const chunkData = () => {
+  if (uploadMode.value === 'file') {
+    chunkFiles();
+  } else if (uploadMode.value === 'url') {
+    chunkUrls();
+  }
+}
 
 </script>
 
@@ -1181,6 +1255,29 @@ onUnmounted(() => {
     }
   }
 }
+
+.url-input {
+  margin-bottom: 20px;
+}
+
+.url-input .ant-textarea {
+  border-color: var(--main-light-3);
+  background-color: #fff;
+  font-family: monospace;
+  resize: vertical;
+}
+
+.url-input .ant-textarea:hover,
+.url-input .ant-textarea:focus {
+  border-color: var(--main-color);
+}
+
+.url-hint {
+  font-size: 13px;
+  color: var(--gray-600);
+  margin-top: 5px;
+  line-height: 1.5;
+}
 </style>
 
 <style lang="less">