feat: 添加自动索引功能,优化文件处理流程

- 在知识库中新增自动索引参数,支持在文件处理后自动触发索引。
- 更新前端界面,允许用户配置自动索引选项,并在处理结果中反馈索引状态。
- 调整定时器刷新间隔,提高数据库信息更新的实时性。
This commit is contained in:
Wenjie Zhang 2025-06-25 02:45:57 +08:00
parent 5aebc248c3
commit 4afbb42f96
6 changed files with 31 additions and 20 deletions

View File

@ -460,6 +460,9 @@ class KnowledgeBase:
self.update_file_status(file_id, "pending_indexing")
file_record['status'] = "pending_indexing" # Ensure status is up-to-date
if params.get("auto_indexing", False):
await self.trigger_file_indexing(db_id, file_id)
except Exception as e:
logger.error(f"处理文件 {file_path} 失败,无法保存待索引块: {e}, {traceback.format_exc()}")
self.update_file_status(file_id, "failed") # Mark file as failed
@ -510,6 +513,9 @@ class KnowledgeBase:
self.update_file_status(file_id, "pending_indexing")
file_record['status'] = "pending_indexing"
if params.get("auto_indexing", False):
await self.trigger_file_indexing(db_id, file_id)
except Exception as e:
logger.error(f"处理URL {url} 失败,无法保存待索引块: {e}, {traceback.format_exc()}")
self.update_file_status(file_id, "failed")

View File

@ -159,7 +159,7 @@ class OtherEmbedding(BaseEmbeddingModel):
self.embed_model_fullname = config.embed_model
self.dimension = self.info.get("dimension", None)
self.model = self.info["name"]
self.api_key = os.getenv(self.info["api_key"], None)
self.api_key = os.getenv(self.info["api_key"], self.info["api_key"])
self.url = get_docker_safe_url(self.info["url"])
assert self.url and self.model, f"URL and model are required. Cur embed model: {config.embed_model}"
self.headers = {
@ -190,6 +190,7 @@ def get_embedding_model():
assert config.embed_model in support_embed_models, f"Unsupported embed model: {config.embed_model}, only support {support_embed_models}"
logger.debug(f"Loading embedding model {config.embed_model}")
if provider == "local":
logger.warning("[DEPRECATED] Local embedding model will be removed in v0.2, please use other embedding models")
model = LocalEmbeddingModel()
elif provider == "zhipu":

View File

@ -22,7 +22,7 @@ class LocalReranker(FlagReranker):
def sigmoid(x):
return 1 / (1 + np.exp(-x))
class SiliconFlowReranker:
class OnlineRerank:
def __init__(self, **kwargs):
model_info = config.reranker_names[config.reranker]
self.url = get_docker_safe_url(model_info["url"])
@ -64,9 +64,10 @@ def get_reranker():
assert config.reranker in support_rerankers, f"Unsupported Reranker: {config.reranker}, only support {support_rerankers}"
provider, model_name = config.reranker.split('/', 1)
if provider == "local":
logger.warning("[DEPRECATED] Local reranker will be removed in v0.2, please use other reranker")
return LocalReranker()
elif provider == "siliconflow":
return SiliconFlowReranker()
return OnlineRerank()
else:
raise ValueError(f"Unsupported Reranker: {config.reranker}, only support {config.reranker_names.keys()}")

View File

@ -111,11 +111,6 @@ MODEL_NAMES:
EMBED_MODEL_INFO:
local/BAAI/bge-m3:
name: BAAI/bge-m3
dimension: 1024
# local_path: /models/BAAI/bge-m3也可以在这里配置
zhipu/zhipu-embedding-2:
name: embedding-2
dimension: 1024
@ -131,25 +126,23 @@ EMBED_MODEL_INFO:
api_key: SILICONFLOW_API_KEY
vllm/Qwen/Qwen3-Embedding-0.6B:
name: qwen3-embedding-0.6b
name: Qwen3-Embedding-0.6B
dimension: 1024
url: http://172.19.13.6:8081/v1/embeddings
url: http://localhost:8081/v1/embeddings
api_key: no_api_key
ollama/nomic-embed-text:
name: nomic-embed-text
url: http://localhost:11434/api/embed
dimension: 768
ollama/bge-m3:
name: bge-m3
url: http://localhost:11434/api/embed
dimension: 1024
RERANKER_LIST:
local/BAAI/bge-reranker-v2-m3:
name: BAAI/bge-reranker-v2-m3
# local_path: /models/BAAI/bge-m3也可以在这里配置
siliconflow/BAAI/bge-reranker-v2-m3:
name: BAAI/bge-reranker-v2-m3
url: https://api.siliconflow.cn/v1/rerank
@ -157,5 +150,5 @@ RERANKER_LIST:
vllm/Qwen/Qwen3-Reranker-0.6B:
name: Qwen/Qwen3-Reranker-0.6B
url: http://172.19.13.6:8081/v1/rerank
url: http://localhost:8081/v1/rerank
api_key: no_api_key

View File

@ -65,6 +65,10 @@
<a-input-number v-model:value="tempChunkParams.chunk_overlap" :min="0" :max="1000" style="width: 100%;" />
<p class="param-description">相邻文本片段间的重叠字符数</p>
</a-form-item>
<a-form-item label="自动索引" name="auto_indexing">
<a-switch v-model:checked="tempChunkParams.auto_indexing" />
<p class="param-description">分块完成后自动开始索引生成向量并写入数据库</p>
</a-form-item>
</a-form>
</div>
</a-modal>
@ -95,7 +99,7 @@
</div>
<div class="config-controls">
<a-button type="dashed" @click="showChunkConfigModal">
<SettingOutlined /> 分块参数 ({{ chunkParams.chunk_size }}/{{ chunkParams.chunk_overlap }})
<SettingOutlined /> 分块参数 ({{ chunkParams.chunk_size }}/{{ chunkParams.chunk_overlap }}{{ chunkParams.auto_indexing ? '/自动索引' : '' }})
</a-button>
</div>
</div>
@ -781,6 +785,7 @@ const chunkParams = ref({
chunk_size: 1000,
chunk_overlap: 200,
enable_ocr: 'disable',
auto_indexing: false,
})
const chunkResults = ref([]);
@ -812,7 +817,8 @@ const chunkFiles = () => {
.then(data => {
console.log('文件处理结果:', data)
if (data.status === 'success') {
message.info(data.message || '文件已提交处理,请稍后在列表刷新查看状态');
const autoIndexingInfo = chunkParams.value.auto_indexing ? ',自动索引已启动' : ',请手动点击索引按钮完成索引';
message.info(data.message + autoIndexingInfo || '文件已提交处理,请稍后在列表刷新查看状态');
fileList.value = []; //
addFilesModalVisible.value = false; //
getDatabaseInfo(); //
@ -850,7 +856,8 @@ const chunkUrls = () => {
.then(data => {
console.log('URL处理结果:', data);
if (data.status === 'success') {
message.success(data.message || 'URL已提交处理请稍后在列表刷新查看状态');
const autoIndexingInfo = chunkParams.value.auto_indexing ? ',自动索引已启动' : ',请手动点击索引按钮完成索引';
message.success(data.message + autoIndexingInfo || 'URL已提交处理请稍后在列表刷新查看状态');
urlList.value = ''; // URL
addFilesModalVisible.value = false; //
getDatabaseInfo(); //
@ -907,7 +914,7 @@ onMounted(() => {
getDatabaseInfo();
state.refreshInterval = setInterval(() => {
getDatabaseInfo();
}, 10000);
}, 1000);
})
// onUnmounted
@ -951,6 +958,7 @@ const chunkConfigModalVisible = ref(false);
const tempChunkParams = ref({
chunk_size: 1000,
chunk_overlap: 200,
auto_indexing: false,
});
//
@ -997,6 +1005,7 @@ const showChunkConfigModal = () => {
tempChunkParams.value = {
chunk_size: chunkParams.value.chunk_size,
chunk_overlap: chunkParams.value.chunk_overlap,
auto_indexing: chunkParams.value.auto_indexing,
};
chunkConfigModalVisible.value = true;
};
@ -1005,6 +1014,7 @@ const showChunkConfigModal = () => {
const handleChunkConfigSubmit = () => {
chunkParams.value.chunk_size = tempChunkParams.value.chunk_size;
chunkParams.value.chunk_overlap = tempChunkParams.value.chunk_overlap;
chunkParams.value.auto_indexing = tempChunkParams.value.auto_indexing;
chunkConfigModalVisible.value = false;
message.success('分块参数配置已更新');
};

View File

@ -96,7 +96,7 @@
<ModelProvidersComponent />
</div>
<div class="setting" v-if="(state.windowWidth <= 520 || state.section ==='path') && userStore.isSuperAdmin">
<h3>本地模型配置</h3>
<h3>本地模型配置将在 v0.2 版本移除对本地模型的支持</h3>
<p>如果是 Docker 启动务必确保在 docker-compose.dev.yaml 中添加了 volumes 映射</p>
<TableConfigComponent
:config="configStore.config?.model_local_paths"