From 4afbb42f96f7f88ca88b67568a69ccb24305c11f Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Wed, 25 Jun 2025 02:45:57 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E7=B4=A2=E5=BC=95=E5=8A=9F=E8=83=BD=EF=BC=8C=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=A4=84=E7=90=86=E6=B5=81=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 在知识库中新增自动索引参数，支持在文件处理后自动触发索引。 - 更新前端界面，允许用户配置自动索引选项，并在处理结果中反馈索引状态。 - 调整定时器刷新间隔，提高数据库信息更新的实时性。 --- src/core/knowledgebase.py | 6 ++++++ src/models/embedding.py | 3 ++- src/models/rerank_model.py | 5 +++-- src/static/models.yaml | 17 +++++------------ web/src/views/DataBaseInfoView.vue | 18 ++++++++++++++---- web/src/views/SettingView.vue | 2 +- 6 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/core/knowledgebase.py b/src/core/knowledgebase.py index 787020c2..7da0c060 100644 --- a/src/core/knowledgebase.py +++ b/src/core/knowledgebase.py @@ -460,6 +460,9 @@ class KnowledgeBase: self.update_file_status(file_id, "pending_indexing") file_record['status'] = "pending_indexing" # Ensure status is up-to-date + if params.get("auto_indexing", False): + await self.trigger_file_indexing(db_id, file_id) + except Exception as e: logger.error(f"处理文件 {file_path} 失败，无法保存待索引块: {e}, {traceback.format_exc()}") self.update_file_status(file_id, "failed") # Mark file as failed @@ -510,6 +513,9 @@ class KnowledgeBase: self.update_file_status(file_id, "pending_indexing") file_record['status'] = "pending_indexing" + if params.get("auto_indexing", False): + await self.trigger_file_indexing(db_id, file_id) + except Exception as e: logger.error(f"处理URL {url} 失败，无法保存待索引块: {e}, {traceback.format_exc()}") self.update_file_status(file_id, "failed") diff --git a/src/models/embedding.py b/src/models/embedding.py index 7fb16b02..6718c887 100644 --- a/src/models/embedding.py +++ b/src/models/embedding.py @@ -159,7 +159,7 @@ class OtherEmbedding(BaseEmbeddingModel): self.embed_model_fullname = config.embed_model self.dimension = self.info.get("dimension", None) self.model = self.info["name"] - self.api_key = os.getenv(self.info["api_key"], None) + self.api_key = os.getenv(self.info["api_key"], self.info["api_key"]) self.url = get_docker_safe_url(self.info["url"]) assert self.url and self.model, f"URL and model are required. Cur embed model: {config.embed_model}" self.headers = { @@ -190,6 +190,7 @@ def get_embedding_model(): assert config.embed_model in support_embed_models, f"Unsupported embed model: {config.embed_model}, only support {support_embed_models}" logger.debug(f"Loading embedding model {config.embed_model}") if provider == "local": + logger.warning("[DEPRECATED] Local embedding model will be removed in v0.2, please use other embedding models") model = LocalEmbeddingModel() elif provider == "zhipu": diff --git a/src/models/rerank_model.py b/src/models/rerank_model.py index f8d00968..85109d91 100644 --- a/src/models/rerank_model.py +++ b/src/models/rerank_model.py @@ -22,7 +22,7 @@ class LocalReranker(FlagReranker): def sigmoid(x): return 1 / (1 + np.exp(-x)) -class SiliconFlowReranker: +class OnlineRerank: def __init__(self, **kwargs): model_info = config.reranker_names[config.reranker] self.url = get_docker_safe_url(model_info["url"]) @@ -64,9 +64,10 @@ def get_reranker(): assert config.reranker in support_rerankers, f"Unsupported Reranker: {config.reranker}, only support {support_rerankers}" provider, model_name = config.reranker.split('/', 1) if provider == "local": + logger.warning("[DEPRECATED] Local reranker will be removed in v0.2, please use other reranker") return LocalReranker() elif provider == "siliconflow": - return SiliconFlowReranker() + return OnlineRerank() else: raise ValueError(f"Unsupported Reranker: {config.reranker}, only support {config.reranker_names.keys()}") diff --git a/src/static/models.yaml b/src/static/models.yaml index 61ff1074..cb8192c7 100644 --- a/src/static/models.yaml +++ b/src/static/models.yaml @@ -111,11 +111,6 @@ MODEL_NAMES: EMBED_MODEL_INFO: - local/BAAI/bge-m3: - name: BAAI/bge-m3 - dimension: 1024 - # local_path: /models/BAAI/bge-m3，也可以在这里配置 - zhipu/zhipu-embedding-2: name: embedding-2 dimension: 1024 @@ -131,25 +126,23 @@ EMBED_MODEL_INFO: api_key: SILICONFLOW_API_KEY vllm/Qwen/Qwen3-Embedding-0.6B: - name: qwen3-embedding-0.6b + name: Qwen3-Embedding-0.6B dimension: 1024 - url: http://172.19.13.6:8081/v1/embeddings + url: http://localhost:8081/v1/embeddings api_key: no_api_key ollama/nomic-embed-text: name: nomic-embed-text + url: http://localhost:11434/api/embed dimension: 768 ollama/bge-m3: name: bge-m3 + url: http://localhost:11434/api/embed dimension: 1024 RERANKER_LIST: - local/BAAI/bge-reranker-v2-m3: - name: BAAI/bge-reranker-v2-m3 - # local_path: /models/BAAI/bge-m3，也可以在这里配置 - siliconflow/BAAI/bge-reranker-v2-m3: name: BAAI/bge-reranker-v2-m3 url: https://api.siliconflow.cn/v1/rerank @@ -157,5 +150,5 @@ RERANKER_LIST: vllm/Qwen/Qwen3-Reranker-0.6B: name: Qwen/Qwen3-Reranker-0.6B - url: http://172.19.13.6:8081/v1/rerank + url: http://localhost:8081/v1/rerank api_key: no_api_key diff --git a/web/src/views/DataBaseInfoView.vue b/web/src/views/DataBaseInfoView.vue index f189701b..620b828d 100644 --- a/web/src/views/DataBaseInfoView.vue +++ b/web/src/views/DataBaseInfoView.vue @@ -65,6 +65,10 @@

相邻文本片段间的重叠字符数

+ + +

分块完成后自动开始索引，生成向量并写入数据库

+ @@ -95,7 +99,7 @@
- 分块参数 ({{ chunkParams.chunk_size }}/{{ chunkParams.chunk_overlap }}) + 分块参数 ({{ chunkParams.chunk_size }}/{{ chunkParams.chunk_overlap }}{{ chunkParams.auto_indexing ? '/自动索引' : '' }})
@@ -781,6 +785,7 @@ const chunkParams = ref({ chunk_size: 1000, chunk_overlap: 200, enable_ocr: 'disable', + auto_indexing: false, }) const chunkResults = ref([]); @@ -812,7 +817,8 @@ const chunkFiles = () => { .then(data => { console.log('文件处理结果:', data) if (data.status === 'success') { - message.info(data.message || '文件已提交处理，请稍后在列表刷新查看状态'); + const autoIndexingInfo = chunkParams.value.auto_indexing ? '，自动索引已启动' : '，请手动点击索引按钮完成索引'; + message.info(data.message + autoIndexingInfo || '文件已提交处理，请稍后在列表刷新查看状态'); fileList.value = []; // 清空已上传文件列表 addFilesModalVisible.value = false; // 关闭弹窗 getDatabaseInfo(); // 刷新数据库信息以显示新文件及其状态 @@ -850,7 +856,8 @@ const chunkUrls = () => { .then(data => { console.log('URL处理结果:', data); if (data.status === 'success') { - message.success(data.message || 'URL已提交处理，请稍后在列表刷新查看状态'); + const autoIndexingInfo = chunkParams.value.auto_indexing ? '，自动索引已启动' : '，请手动点击索引按钮完成索引'; + message.success(data.message + autoIndexingInfo || 'URL已提交处理，请稍后在列表刷新查看状态'); urlList.value = ''; // 清空URL输入 addFilesModalVisible.value = false; // 关闭弹窗 getDatabaseInfo(); // 刷新数据库信息以显示新文件及其状态 @@ -907,7 +914,7 @@ onMounted(() => { getDatabaseInfo(); state.refreshInterval = setInterval(() => { getDatabaseInfo(); - }, 10000); + }, 1000); }) // 添加 onUnmounted 钩子，在组件卸载时清除定时器 @@ -951,6 +958,7 @@ const chunkConfigModalVisible = ref(false); const tempChunkParams = ref({ chunk_size: 1000, chunk_overlap: 200, + auto_indexing: false, }); // 添加文件弹窗 @@ -997,6 +1005,7 @@ const showChunkConfigModal = () => { tempChunkParams.value = { chunk_size: chunkParams.value.chunk_size, chunk_overlap: chunkParams.value.chunk_overlap, + auto_indexing: chunkParams.value.auto_indexing, }; chunkConfigModalVisible.value = true; }; @@ -1005,6 +1014,7 @@ const showChunkConfigModal = () => { const handleChunkConfigSubmit = () => { chunkParams.value.chunk_size = tempChunkParams.value.chunk_size; chunkParams.value.chunk_overlap = tempChunkParams.value.chunk_overlap; + chunkParams.value.auto_indexing = tempChunkParams.value.auto_indexing; chunkConfigModalVisible.value = false; message.success('分块参数配置已更新'); }; diff --git a/web/src/views/SettingView.vue b/web/src/views/SettingView.vue index 0deddca4..0f04eb02 100644 --- a/web/src/views/SettingView.vue +++ b/web/src/views/SettingView.vue @@ -96,7 +96,7 @@
-

本地模型配置

+

本地模型配置（将在 v0.2 版本移除对本地模型的支持）

如果是 Docker 启动，务必确保在 docker-compose.dev.yaml 中添加了 volumes 映射。