feat: 更新依赖和优化数据处理接口

-  优化了数据路由器中的文件和 URL 分块处理逻辑，增加了错误处理和日志记录。
- 更新了前端 API 接口以支持新的文件索引流程，移除了过时的接口。

pyproject.toml

@@ -17,6 +17,7 @@ dependencies = [
     "langchain-together>=0.3.0",
     "langgraph>=0.3.34",
     "langgraph-checkpoint-sqlite>=2.0.7",
+    "langgraph-cli[inmem]>=0.1.54",
     "langsmith>=0.3.37",
     "llama-index>=0.12.33",
     "llama-index-readers-file>=0.4.7",

server/models/kb_models.py

@@ -59,6 +59,11 @@ class KnowledgeFile(Base):
     database = relationship("KnowledgeDatabase", back_populates="files")
     nodes = relationship("KnowledgeNode", back_populates="file", cascade="all, delete-orphan")
 
+    @property
+    def computed_node_count(self):
+        """动态计算节点数量"""
+        return len(self.nodes) if self.nodes is not None else 0
+
     def to_dict(self):
         """转换为字典格式"""
         result = {
@@ -67,6 +72,7 @@ class KnowledgeFile(Base):
             "path": self.path,
             "type": self.file_type,
             "status": self.status,
+            "node_count": self.computed_node_count,  # 使用计算属性
             "created_at": self.created_at.timestamp() if self.created_at else time.time()
         }
 

server/routers/data_router.py

@@ -52,36 +52,42 @@ async def query_test(query: str = Body(...), meta: dict = Body(...), current_use
     return result
 
 @data.post("/file-to-chunk")
-async def file_to_chunk(files: list[str] = Body(...), params: dict = Body(...), current_user: User = Depends(get_admin_user)):
+async def file_to_chunk(db_id: str = Body(...), files: list[str] = Body(...), params: dict = Body(...), current_user: User = Depends(get_admin_user)):
-    logger.debug(f"File to chunk: {files} {params=}")
+    logger.debug(f"File to chunk for db_id {db_id}: {files} {params=}")
-    result = await knowledge_base.file_to_chunk(files, params=params)
+    try:
-    return result
+        processed_files = await knowledge_base.save_files_for_pending_indexing(db_id, files, params)
+        return {"message": "Files processed and pending indexing", "files": processed_files, "status": "success"}
+    except Exception as e:
+        logger.error(f"Failed to process files for pending indexing: {e}, {traceback.format_exc()}")
+        return {"message": f"Failed to process files for pending indexing: {e}", "status": "failed"}
 
 @data.post("/url-to-chunk")
-async def url_to_chunk(urls: list[str] = Body(...), params: dict = Body(...), current_user: User = Depends(get_admin_user)):
+async def url_to_chunk(db_id: str = Body(...), urls: list[str] = Body(...), params: dict = Body(...), current_user: User = Depends(get_admin_user)):
-    logger.debug(f"Url to chunk: {urls}")
+    logger.debug(f"Url to chunk for db_id {db_id}: {urls} {params=}")
-    result = await knowledge_base.url_to_chunk(urls, params=params)
+    try:
-    return result
+        processed_urls = await knowledge_base.save_urls_for_pending_indexing(db_id, urls, params)
+        return {"message": "URLs processed and pending indexing", "urls": processed_urls, "status": "success"}
+    except Exception as e:
+        logger.error(f"Failed to process URLs for pending indexing: {e}, {traceback.format_exc()}")
+        return {"message": f"Failed to process URLs for pending indexing: {e}", "status": "failed"}
 
 @data.post("/add-by-file")
 async def create_document_by_file(db_id: str = Body(...), files: list[str] = Body(...), current_user: User = Depends(get_admin_user)):
-    logger.debug(f"Add document in {db_id} by file: {files}")
+    raise ValueError("This method is deprecated. Use /file-to-chunk and /index-file instead.")
-    try:
-        await knowledge_base.add_files(db_id, files)
-        return {"message": "文件添加完成", "status": "success"}
-    except Exception as e:
-        logger.error(f"添加文件失败: {e}, {traceback.format_exc()}")
-        return {"message": f"添加文件失败: {e}", "status": "failed"}
 
 @data.post("/add-by-chunks")
 async def add_by_chunks(db_id: str = Body(...), file_chunks: dict = Body(...), current_user: User = Depends(get_admin_user)):
-    # logger.debug(f"Add chunks in {db_id}: {len(file_chunks)} chunks")
+    raise ValueError("This method is deprecated. Use /file-to-chunk and /index-file instead.")
+
+@data.post("/index-file")
+async def index_file(db_id: str = Body(...), file_id: str = Body(...), current_user: User = Depends(get_admin_user)):
+    logger.debug(f"Indexing file_id {file_id} in db_id {db_id}")
     try:
-        await knowledge_base.add_chunks(db_id, file_chunks)
+        result = await knowledge_base.trigger_file_indexing(db_id, file_id)
-        return {"message": "分块添加完成", "status": "success"}
+        return {"message": f"File {file_id} indexing initiated", "details": result, "status": "success"}
     except Exception as e:
-        logger.error(f"添加分块失败: {e}, {traceback.format_exc()}")
+        logger.error(f"Failed to index file {file_id}: {e}, {traceback.format_exc()}")
-        return {"message": f"添加分块失败: {e}", "status": "failed"}
+        return {"message": f"Failed to index file {file_id}: {e}", "status": "failed"}
 
 @data.get("/info")
 async def get_database_info(db_id: str, current_user: User = Depends(get_admin_user)):

src/core/retriever.py

@@ -107,7 +107,7 @@ class Retriever:
         rw_query = self.rewrite_query(query, history, refs)
 
         logger.debug(f"{meta=}")
-        query_result = knowledge_base.query(query=rw_query,
+        query_result = knowledge_base.query(query_text=rw_query,
                                             db_id=db_id,
                                             distance_threshold=meta.get("distanceThreshold", 0.5),
                                             rerank_threshold=meta.get("rerankThreshold", 0.1),

src/utils/__init__.py

@@ -1,5 +1,5 @@
 import time
-import random
+import hashlib
 import os
 from src.utils.logging_config import logger
 
@@ -22,14 +22,28 @@ def is_text_pdf(pdf_path):
     # 如果超过50%的页面有文本内容，则认为是文本PDF
     return text_ratio > 0.5
 
-def hashstr(input_string, length=8, with_salt=False):
+def hashstr(input_string, length=None, with_salt=False):
-    import hashlib
+    """生成字符串的哈希值
-    # 添加时间戳作为干扰
+    Args:
-    if with_salt:
+        input_string: 输入字符串
-        input_string += str(time.time() + random.random())
+        length: 截取长度，默认为None，表示不截取
+        with_salt: 是否加盐，默认为False
+    """
+    try:
+        # 尝试直接编码
+        encoded_string = str(input_string).encode('utf-8')
+    except UnicodeEncodeError:
+        # 如果编码失败，替换无效字符
+        encoded_string = str(input_string).encode('utf-8', errors='replace')
 
-    hash = hashlib.md5(str(input_string).encode()).hexdigest()
+    if with_salt:
+        salt = str(time.time())
+        encoded_string = (encoded_string.decode('utf-8') + salt).encode('utf-8')
+
+    hash = hashlib.md5(encoded_string).hexdigest()
+    if length:
         return hash[:length]
+    return hash
 
 
 def get_docker_safe_url(base_url):

web/src/apis/admin_api.js

@@ -150,15 +150,35 @@ export const knowledgeBaseApi = {
   },
 
   /**
-   * 将文件分块
+   * 将文件分块并保存文件记录以待索引
-   * @param {Object} data - 分块参数
+   * @param {Object} data - 包含 db_id, files (路径列表), params (分块参数)
-   * @returns {Promise} - 分块结果
+   * @returns {Promise} - 处理结果
    */
-  fileToChunk: async (data) => {
+  fileToChunk: async (data) => { // data: { db_id, files, params }
     checkAdminPermission()
     return apiPost('/api/data/file-to-chunk', data, {}, true)
   },
 
+  /**
+   * 将URL分块并保存文件记录以待索引
+   * @param {Object} data - 包含 db_id, urls (链接列表), params (分块参数)
+   * @returns {Promise} - 处理结果
+   */
+  urlToChunk: async (data) => { // data: { db_id, urls, params }
+    checkAdminPermission()
+    return apiPost('/api/data/url-to-chunk', data, {}, true)
+  },
+
+  /**
+   * 触发指定文件的索引过程
+   * @param {Object} data - 包含 db_id, file_id
+   * @returns {Promise} - 索引启动结果
+   */
+  indexFile: async (data) => { // data: { db_id, file_id }
+    checkAdminPermission()
+    return apiPost('/api/data/index-file', data, {}, true)
+  },
+
   /**
    * 将分块添加到数据库
    * @param {Object} data - 包含db_id和file_chunks的数据
@@ -190,16 +210,6 @@ export const knowledgeBaseApi = {
     return apiGet(`/api/data/document?db_id=${dbId}&file_id=${fileId}`, {}, true)
   },
 
-  /**
-   * 将URL转换为分块
-   * @param {Object} data - 分块参数
-   * @returns {Promise} - 分块结果
-   */
-  urlToChunk: async (data) => {
-    checkAdminPermission()
-    return apiPost('/api/data/url-to-chunk', data, {}, true)
-  },
-
   /**
    * 更新知识库信息
    * @param {string} dbId - 知识库ID

web/src/views/DataBaseInfoView.vue

@@ -42,60 +42,11 @@
       <a-tab-pane key="files">
         <template #tab><span><ReadOutlined />文件列表</span></template>
         <div class="db-tab-container">
-          <div class="actions">
+          <div class="actions" style="display: flex; gap: 10px;">
+            <a-button type="primary" @click="handleShowAddFilesBlock" :loading="state.refrashing" :icon="h(PlusOutlined)">添加文件</a-button>
             <a-button @click="handleRefresh" :loading="state.refrashing">刷新</a-button>
           </div>
-          <a-table :columns="columns" :data-source="Object.values(database.files || {})" row-key="file_id" class="my-table">
+          <div class="upload-section" v-if="state.showAddFilesBlock">
-            <template #bodyCell="{ column, text, record }">
-              <template v-if="column.key === 'filename'">
-                <a-button class="main-btn" type="link" @click="openFileDetail(record)">{{ text }}</a-button>
-              </template>
-              <template v-else-if="column.key === 'type'">
-                <span :class="['span-type', text]">{{ text?.toUpperCase() }}</span>
-              </template>
-              <template v-else-if="column.key === 'status' && text === 'done'">
-                <CheckCircleFilled style="color: #41A317;"/>
-              </template>
-              <template v-else-if="column.key === 'status' && text === 'failed'">
-                <CloseCircleFilled style="color: #FF4D4F ;"/>
-              </template>
-              <template v-else-if="column.key === 'status' && text === 'processing'">
-                <HourglassFilled style="color: #1677FF;"/>
-              </template>
-              <template v-else-if="column.key === 'status' && text === 'waiting'">
-                <ClockCircleFilled style="color: #FFCD43;"/>
-              </template>
-              <template v-else-if="column.key === 'action'">
-                <a-button class="del-btn" type="link"
-                  @click="deleteFile(text)"
-                  :disabled="state.lock || record.status === 'processing' || record.status === 'waiting' "
-                  >删除
-                </a-button>
-              </template>
-              <span v-else-if="column.key === 'created_at'">{{ formatRelativeTime(Math.round(text*1000)) }}</span>
-              <span v-else>{{ text }}</span>
-            </template>
-          </a-table>
-          <a-drawer
-            width="50%"
-            v-model:open="state.drawer"
-            class="custom-class"
-            :title="selectedFile?.filename || '文件详情'"
-            placement="right"
-            @after-open-change="afterOpenChange"
-          >
-            <h2>共 {{ selectedFile?.lines?.length || 0 }} 个片段</h2>
-            <p v-for="line in selectedFile?.lines || []" :key="line.id" class="line-text">
-              {{ line.text }}
-            </p>
-          </a-drawer>
-        </div>
-      </a-tab-pane>
-
-      <a-tab-pane key="add">
-        <template #tab><span><CloudUploadOutlined />添加文件</span></template>
-        <div class="db-tab-container">
-          <div class="upload-section">
             <div class="upload-sidebar">
               <div class="chunking-params">
                 <div class="params-info">
@@ -184,30 +135,67 @@
               </div>
             </div>
           </div>
-
+          <a-table :columns="columns" :data-source="Object.values(database.files || {})" row-key="file_id" class="my-table">
-          <!-- 分块结果预览区域 -->
+            <template #bodyCell="{ column, text, record }">
-          <div class="chunk-preview" v-if="chunkResults.length > 0">
+              <template v-if="column.key === 'filename'">
-            <div class="preview-header">
+                <a-tooltip :title="record.file_id" placement="right">
-              <h3>分块预览 (共 {{ chunkResults.length }} 个文件，{{ getTotalChunks() }} 个分块)</h3>
+                  <a-button class="main-btn" type="link" @click="openFileDetail(record)">{{ text }}</a-button>
+                </a-tooltip>
+              </template>
+              <template v-else-if="column.key === 'type'">
+                <span :class="['span-type', text]">{{ text?.toUpperCase() }}</span>
+              </template>
+              <template v-else-if="column.key === 'status' && text === 'done'">
+                <CheckCircleFilled style="color: #41A317;"/>
+              </template>
+              <template v-else-if="column.key === 'status' && text === 'failed'">
+                <CloseCircleFilled style="color: #FF4D4F ;"/>
+              </template>
+              <template v-else-if="column.key === 'status' && text === 'processing'">
+                <HourglassFilled style="color: #1677FF;"/>
+              </template>
+              <template v-else-if="column.key === 'status' && text === 'waiting'">
+                <ClockCircleFilled style="color: #FFCD43;"/>
+              </template>
+              <template v-else-if="column.key === 'status' && text === 'pending_indexing'">
+                <HddOutlined style="color: #FFCD43;"/>
+              </template>
+              <template v-else-if="column.key === 'action'">
+                <div style="display: flex; gap: 10px;">
                   <a-button
-                type="primary"
+                    v-if="record.status === 'pending_indexing'"
-                @click="addToDatabase"
+                    type="link"
-                :loading="state.adding"
+                    @click="handleIndexFile(record.file_id)"
+                    :loading="state.indexingFile === record.file_id"
+                    :disabled="state.lock || state.indexingFile === record.file_id"
                   >
-                添加到数据库
+                    索引
+                  </a-button>
+                  <a-button class="del-btn" type="link"
+                    @click="deleteFile(text)"
+                    :disabled="state.lock || record.status === 'processing' || record.status === 'waiting' || state.indexingFile === record.file_id"
+                    >
+                    删除
                   </a-button>
                 </div>
-
+              </template>
-            <a-collapse v-model:activeKey="activeFileKeys">
+              <span v-else-if="column.key === 'created_at'">{{ formatRelativeTime(Math.round(text*1000)) }}</span>
-              <a-collapse-panel v-for="(file, fileIdx) in chunkResults" :key="fileIdx" :header="file.filename + ' (' + file.nodes.length + ' 个分块)'">
+              <span v-else>{{ text }}</span>
-                <div id="result-cards" class="result-cards">
+            </template>
-                  <div v-for="(chunk, index) in file.nodes" :key="index" class="chunk">
+          </a-table>
-                    <p><strong>#{{ index + 1 }}</strong> {{ chunk.text }}</p>
+          <a-drawer
-                  </div>
+            width="50%"
-                </div>
+            v-model:open="state.drawer"
-              </a-collapse-panel>
+            class="custom-class"
-            </a-collapse>
+            :title="selectedFile?.filename || '文件详情'"
-          </div>
+            placement="right"
+            @after-open-change="afterOpenChange"
+          >
+            <h2>共 {{ selectedFile?.lines?.length || 0 }} 个片段</h2>
+            <p v-for="line in selectedFile?.lines || []" :key="line.id" class="line-text">
+              {{ line.text }}
+            </p>
+          </a-drawer>
         </div>
       </a-tab-pane>
 
@@ -350,7 +338,10 @@ import {
   FileOutlined,
   LinkOutlined,
   EditOutlined,
+  PlusOutlined,
+  HddOutlined,
 } from '@ant-design/icons-vue'
+import { h } from 'vue';
 
 
 const route = useRoute();
@@ -376,6 +367,8 @@ const state = reactive({
   drawer: false,
   refreshInterval: null,
   curPage: "files",
+  indexingFile: null,
+  showAddFilesBlock: false,
 });
 
 const meta = reactive({
@@ -505,6 +498,10 @@ const handleRefresh = () => {
   })
 }
 
+const handleShowAddFilesBlock = () => {
+  state.showAddFilesBlock = !state.showAddFilesBlock
+}
+
 const deleteDatabse = () => {
   Modal.confirm({
     title: '删除数据库',
@@ -652,7 +649,7 @@ const getTotalChunks = () => {
   return chunkResults.value.reduce((total, file) => total + file.nodes.length, 0);
 }
 
-// 分块预览
+// "生成分块" - 修改后的逻辑
 const chunkFiles = () => {
   console.log(fileList.value)
   const files = fileList.value.filter(file => file.status === 'done').map(file => file.response.file_path)
@@ -665,28 +662,34 @@ const chunkFiles = () => {
 
   state.loading = true
 
-  // 调用file-to-chunk接口获取分块信息
   knowledgeBaseApi.fileToChunk({
+    db_id: databaseId.value, // 添加 db_id
     files: files,
     params: chunkParams.value
   })
   .then(data => {
-    console.log('文件分块信息:', data)
+    console.log('文件处理结果:', data)
-    chunkResults.value = Object.values(data);
+    if (data.status === 'success') {
-    activeFileKeys.value = chunkResults.value.length > 0 ? [0] : []; // 默认展开第一个文件
+      message.success(data.message || '文件已提交处理，请稍后在列表刷新查看状态');
+      fileList.value = []; // 清空已上传文件列表
+      // chunkResults.value = []; // 清空旧的预览结果
+      // activeFileKeys.value = []; // 清空旧的预览结果
+      getDatabaseInfo(); // 刷新数据库信息以显示新文件及其状态
+    } else {
+      message.error(data.message || '文件处理失败');
+    }
   })
   .catch(error => {
     console.error(error)
-    message.error(error.message)
+    message.error(error.message || '文件处理请求失败')
   })
   .finally(() => {
     state.loading = false
   })
 }
 
-// 分块预览
+// "生成分块" - 修改后的逻辑
 const chunkUrls = () => {
-  // 分割并过滤URL列表
   const urls = urlList.value.split('\n')
     .map(url => url.trim())
     .filter(url => url.length > 0 && (url.startsWith('http://') || url.startsWith('https://')));
@@ -698,67 +701,74 @@ const chunkUrls = () => {
 
   state.loading = true;
 
-  // 调用url-to-chunk接口获取分块信息
   knowledgeBaseApi.urlToChunk({
+    db_id: databaseId.value, // 添加 db_id
     urls: urls,
     params: chunkParams.value
   })
   .then(data => {
-    console.log('URL分块信息:', data);
+    console.log('URL处理结果:', data);
-    chunkResults.value = Object.values(data);
+    if (data.status === 'success') {
-    activeFileKeys.value = chunkResults.value.length > 0 ? [0] : []; // 默认展开第一个
+      message.success(data.message || 'URL已提交处理，请稍后在列表刷新查看状态');
+      urlList.value = ''; // 清空URL输入
+      // chunkResults.value = []; // 清空旧的预览结果
+      // activeFileKeys.value = []; // 清空旧的预览结果
+      getDatabaseInfo(); // 刷新数据库信息以显示新文件及其状态
+    } else {
+      message.error(data.message || 'URL处理失败');
+    }
   })
   .catch(error => {
     console.error(error);
-    message.error(error.message || '处理URL失败');
+    message.error(error.message || '处理URL请求失败');
   })
   .finally(() => {
     state.loading = false;
   });
 };
 
-// 添加到数据库
+// 添加到数据库 - 此函数逻辑将被新的 "索引" 按钮替代或移除
 const addToDatabase = () => {
-  if (chunkResults.value.length === 0) {
+  // if (chunkResults.value.length === 0) {
-    message.error('没有可添加的分块')
+  //   message.error('没有可添加的分块')
-    return
+  //   return
-  }
+  // }
+  // state.adding = true
+  // state.lock = true
 
-  state.adding = true
+  // // 转换为API需要的格式
-  state.lock = true
+  // const fileChunks = {};
+  // chunkResults.value.forEach(file => {
+  //   fileChunks[file.file_id] = file;
+  // });
 
-  // 转换为API需要的格式
+  // // 调用add-by-chunks接口将分块添加到数据库
-  const fileChunks = {};
+  // knowledgeBaseApi.addByChunks({
-  chunkResults.value.forEach(file => {
+  //   db_id: databaseId.value,
-    fileChunks[file.file_id] = file;
+  //   file_chunks: fileChunks
-  });
+  // })
+  // .then(data => {
+  //   console.log(data)
 
-  // 调用add-by-chunks接口将分块添加到数据库
+  //   if (data.status === 'failed') {
-  knowledgeBaseApi.addByChunks({
+  //     message.error(data.message)
-    db_id: databaseId.value,
+  //   } else {
-    file_chunks: fileChunks
+  //     message.success(data.message)
-  })
+  //     fileList.value = []
-  .then(data => {
+  //     chunkResults.value = []
-    console.log(data)
+  //     activeFileKeys.value = []
-
+  //     getDatabaseInfo() // 刷新列表
-    if (data.status === 'failed') {
+  //   }
-      message.error(data.message)
+  // })
-    } else {
+  // .catch(error => {
-      message.success(data.message)
+  //   console.error(error)
-      fileList.value = []
+  //   message.error(error.message)
-      chunkResults.value = []
+  // })
-      activeFileKeys.value = []
+  // .finally(() => {
-    }
+  //   state.adding = false
-  })
+  //   state.lock = false
-  .catch(error => {
+  // })
-    console.error(error)
+  message.info("此功能已通过新的索引流程处理。文件分块后将自动出现在列表中，可单独进行索引。");
-    message.error(error.message)
-  })
-  .finally(() => {
-    getDatabaseInfo()
-    state.adding = false
-    state.lock = false
-  })
 }
 
 const addDocumentByFile = () => {
@@ -881,6 +891,46 @@ const updateDatabaseInfo = async () => {
   }
 };
 
+const handleIndexFile = (fileId) => {
+  if (!fileId) {
+    message.error('无效的文件ID');
+    return;
+  }
+  Modal.confirm({
+    title: '开始索引文件',
+    content: `确定要开始索引文件 ${fileId} 吗？该操作可能需要一些时间。`,
+    okText: '确认索引',
+    cancelText: '取消',
+    onOk: () => {
+      state.indexingFile = fileId; // Set loading state for this specific file
+      state.lock = true;
+      knowledgeBaseApi.indexFile({
+        db_id: databaseId.value,
+        file_id: fileId
+      })
+      .then(response => {
+        if (response.status === 'success') {
+          message.success(response.message || `文件 ${fileId} 已开始索引。`);
+          getDatabaseInfo(); // Refresh to update status
+        } else {
+          message.error(response.message || `文件 ${fileId} 索引启动失败。`);
+        }
+      })
+      .catch(error => {
+        console.error(`索引文件 ${fileId} 失败:`, error);
+        message.error(error.message || `索引文件 ${fileId} 时发生错误。`);
+      })
+      .finally(() => {
+        state.indexingFile = null; // Reset loading state for this file
+        state.lock = false;
+      });
+    },
+    onCancel: () => {
+      console.log('取消索引');
+    },
+  });
+};
+
 </script>
 
 <style lang="less" scoped>