feat: 添加自动索引功能,优化文件处理流程

- 在知识库中新增自动索引参数,支持在文件处理后自动触发索引。
- 更新前端界面,允许用户配置自动索引选项,并在处理结果中反馈索引状态。
- 调整定时器刷新间隔,提高数据库信息更新的实时性。
This commit is contained in:
Wenjie Zhang 2025-06-25 02:45:57 +08:00
parent 5aebc248c3
commit 4afbb42f96
6 changed files with 31 additions and 20 deletions

View File

@ -460,6 +460,9 @@ class KnowledgeBase:
self.update_file_status(file_id, "pending_indexing") self.update_file_status(file_id, "pending_indexing")
file_record['status'] = "pending_indexing" # Ensure status is up-to-date file_record['status'] = "pending_indexing" # Ensure status is up-to-date
if params.get("auto_indexing", False):
await self.trigger_file_indexing(db_id, file_id)
except Exception as e: except Exception as e:
logger.error(f"处理文件 {file_path} 失败,无法保存待索引块: {e}, {traceback.format_exc()}") logger.error(f"处理文件 {file_path} 失败,无法保存待索引块: {e}, {traceback.format_exc()}")
self.update_file_status(file_id, "failed") # Mark file as failed self.update_file_status(file_id, "failed") # Mark file as failed
@ -510,6 +513,9 @@ class KnowledgeBase:
self.update_file_status(file_id, "pending_indexing") self.update_file_status(file_id, "pending_indexing")
file_record['status'] = "pending_indexing" file_record['status'] = "pending_indexing"
if params.get("auto_indexing", False):
await self.trigger_file_indexing(db_id, file_id)
except Exception as e: except Exception as e:
logger.error(f"处理URL {url} 失败,无法保存待索引块: {e}, {traceback.format_exc()}") logger.error(f"处理URL {url} 失败,无法保存待索引块: {e}, {traceback.format_exc()}")
self.update_file_status(file_id, "failed") self.update_file_status(file_id, "failed")

View File

@ -159,7 +159,7 @@ class OtherEmbedding(BaseEmbeddingModel):
self.embed_model_fullname = config.embed_model self.embed_model_fullname = config.embed_model
self.dimension = self.info.get("dimension", None) self.dimension = self.info.get("dimension", None)
self.model = self.info["name"] self.model = self.info["name"]
self.api_key = os.getenv(self.info["api_key"], None) self.api_key = os.getenv(self.info["api_key"], self.info["api_key"])
self.url = get_docker_safe_url(self.info["url"]) self.url = get_docker_safe_url(self.info["url"])
assert self.url and self.model, f"URL and model are required. Cur embed model: {config.embed_model}" assert self.url and self.model, f"URL and model are required. Cur embed model: {config.embed_model}"
self.headers = { self.headers = {
@ -190,6 +190,7 @@ def get_embedding_model():
assert config.embed_model in support_embed_models, f"Unsupported embed model: {config.embed_model}, only support {support_embed_models}" assert config.embed_model in support_embed_models, f"Unsupported embed model: {config.embed_model}, only support {support_embed_models}"
logger.debug(f"Loading embedding model {config.embed_model}") logger.debug(f"Loading embedding model {config.embed_model}")
if provider == "local": if provider == "local":
logger.warning("[DEPRECATED] Local embedding model will be removed in v0.2, please use other embedding models")
model = LocalEmbeddingModel() model = LocalEmbeddingModel()
elif provider == "zhipu": elif provider == "zhipu":

View File

@ -22,7 +22,7 @@ class LocalReranker(FlagReranker):
def sigmoid(x): def sigmoid(x):
return 1 / (1 + np.exp(-x)) return 1 / (1 + np.exp(-x))
class SiliconFlowReranker: class OnlineRerank:
def __init__(self, **kwargs): def __init__(self, **kwargs):
model_info = config.reranker_names[config.reranker] model_info = config.reranker_names[config.reranker]
self.url = get_docker_safe_url(model_info["url"]) self.url = get_docker_safe_url(model_info["url"])
@ -64,9 +64,10 @@ def get_reranker():
assert config.reranker in support_rerankers, f"Unsupported Reranker: {config.reranker}, only support {support_rerankers}" assert config.reranker in support_rerankers, f"Unsupported Reranker: {config.reranker}, only support {support_rerankers}"
provider, model_name = config.reranker.split('/', 1) provider, model_name = config.reranker.split('/', 1)
if provider == "local": if provider == "local":
logger.warning("[DEPRECATED] Local reranker will be removed in v0.2, please use other reranker")
return LocalReranker() return LocalReranker()
elif provider == "siliconflow": elif provider == "siliconflow":
return SiliconFlowReranker() return OnlineRerank()
else: else:
raise ValueError(f"Unsupported Reranker: {config.reranker}, only support {config.reranker_names.keys()}") raise ValueError(f"Unsupported Reranker: {config.reranker}, only support {config.reranker_names.keys()}")

View File

@ -111,11 +111,6 @@ MODEL_NAMES:
EMBED_MODEL_INFO: EMBED_MODEL_INFO:
local/BAAI/bge-m3:
name: BAAI/bge-m3
dimension: 1024
# local_path: /models/BAAI/bge-m3也可以在这里配置
zhipu/zhipu-embedding-2: zhipu/zhipu-embedding-2:
name: embedding-2 name: embedding-2
dimension: 1024 dimension: 1024
@ -131,25 +126,23 @@ EMBED_MODEL_INFO:
api_key: SILICONFLOW_API_KEY api_key: SILICONFLOW_API_KEY
vllm/Qwen/Qwen3-Embedding-0.6B: vllm/Qwen/Qwen3-Embedding-0.6B:
name: qwen3-embedding-0.6b name: Qwen3-Embedding-0.6B
dimension: 1024 dimension: 1024
url: http://172.19.13.6:8081/v1/embeddings url: http://localhost:8081/v1/embeddings
api_key: no_api_key api_key: no_api_key
ollama/nomic-embed-text: ollama/nomic-embed-text:
name: nomic-embed-text name: nomic-embed-text
url: http://localhost:11434/api/embed
dimension: 768 dimension: 768
ollama/bge-m3: ollama/bge-m3:
name: bge-m3 name: bge-m3
url: http://localhost:11434/api/embed
dimension: 1024 dimension: 1024
RERANKER_LIST: RERANKER_LIST:
local/BAAI/bge-reranker-v2-m3:
name: BAAI/bge-reranker-v2-m3
# local_path: /models/BAAI/bge-m3也可以在这里配置
siliconflow/BAAI/bge-reranker-v2-m3: siliconflow/BAAI/bge-reranker-v2-m3:
name: BAAI/bge-reranker-v2-m3 name: BAAI/bge-reranker-v2-m3
url: https://api.siliconflow.cn/v1/rerank url: https://api.siliconflow.cn/v1/rerank
@ -157,5 +150,5 @@ RERANKER_LIST:
vllm/Qwen/Qwen3-Reranker-0.6B: vllm/Qwen/Qwen3-Reranker-0.6B:
name: Qwen/Qwen3-Reranker-0.6B name: Qwen/Qwen3-Reranker-0.6B
url: http://172.19.13.6:8081/v1/rerank url: http://localhost:8081/v1/rerank
api_key: no_api_key api_key: no_api_key

View File

@ -65,6 +65,10 @@
<a-input-number v-model:value="tempChunkParams.chunk_overlap" :min="0" :max="1000" style="width: 100%;" /> <a-input-number v-model:value="tempChunkParams.chunk_overlap" :min="0" :max="1000" style="width: 100%;" />
<p class="param-description">相邻文本片段间的重叠字符数</p> <p class="param-description">相邻文本片段间的重叠字符数</p>
</a-form-item> </a-form-item>
<a-form-item label="自动索引" name="auto_indexing">
<a-switch v-model:checked="tempChunkParams.auto_indexing" />
<p class="param-description">分块完成后自动开始索引生成向量并写入数据库</p>
</a-form-item>
</a-form> </a-form>
</div> </div>
</a-modal> </a-modal>
@ -95,7 +99,7 @@
</div> </div>
<div class="config-controls"> <div class="config-controls">
<a-button type="dashed" @click="showChunkConfigModal"> <a-button type="dashed" @click="showChunkConfigModal">
<SettingOutlined /> 分块参数 ({{ chunkParams.chunk_size }}/{{ chunkParams.chunk_overlap }}) <SettingOutlined /> 分块参数 ({{ chunkParams.chunk_size }}/{{ chunkParams.chunk_overlap }}{{ chunkParams.auto_indexing ? '/自动索引' : '' }})
</a-button> </a-button>
</div> </div>
</div> </div>
@ -781,6 +785,7 @@ const chunkParams = ref({
chunk_size: 1000, chunk_size: 1000,
chunk_overlap: 200, chunk_overlap: 200,
enable_ocr: 'disable', enable_ocr: 'disable',
auto_indexing: false,
}) })
const chunkResults = ref([]); const chunkResults = ref([]);
@ -812,7 +817,8 @@ const chunkFiles = () => {
.then(data => { .then(data => {
console.log('文件处理结果:', data) console.log('文件处理结果:', data)
if (data.status === 'success') { if (data.status === 'success') {
message.info(data.message || '文件已提交处理,请稍后在列表刷新查看状态'); const autoIndexingInfo = chunkParams.value.auto_indexing ? ',自动索引已启动' : ',请手动点击索引按钮完成索引';
message.info(data.message + autoIndexingInfo || '文件已提交处理,请稍后在列表刷新查看状态');
fileList.value = []; // fileList.value = []; //
addFilesModalVisible.value = false; // addFilesModalVisible.value = false; //
getDatabaseInfo(); // getDatabaseInfo(); //
@ -850,7 +856,8 @@ const chunkUrls = () => {
.then(data => { .then(data => {
console.log('URL处理结果:', data); console.log('URL处理结果:', data);
if (data.status === 'success') { if (data.status === 'success') {
message.success(data.message || 'URL已提交处理请稍后在列表刷新查看状态'); const autoIndexingInfo = chunkParams.value.auto_indexing ? ',自动索引已启动' : ',请手动点击索引按钮完成索引';
message.success(data.message + autoIndexingInfo || 'URL已提交处理请稍后在列表刷新查看状态');
urlList.value = ''; // URL urlList.value = ''; // URL
addFilesModalVisible.value = false; // addFilesModalVisible.value = false; //
getDatabaseInfo(); // getDatabaseInfo(); //
@ -907,7 +914,7 @@ onMounted(() => {
getDatabaseInfo(); getDatabaseInfo();
state.refreshInterval = setInterval(() => { state.refreshInterval = setInterval(() => {
getDatabaseInfo(); getDatabaseInfo();
}, 10000); }, 1000);
}) })
// onUnmounted // onUnmounted
@ -951,6 +958,7 @@ const chunkConfigModalVisible = ref(false);
const tempChunkParams = ref({ const tempChunkParams = ref({
chunk_size: 1000, chunk_size: 1000,
chunk_overlap: 200, chunk_overlap: 200,
auto_indexing: false,
}); });
// //
@ -997,6 +1005,7 @@ const showChunkConfigModal = () => {
tempChunkParams.value = { tempChunkParams.value = {
chunk_size: chunkParams.value.chunk_size, chunk_size: chunkParams.value.chunk_size,
chunk_overlap: chunkParams.value.chunk_overlap, chunk_overlap: chunkParams.value.chunk_overlap,
auto_indexing: chunkParams.value.auto_indexing,
}; };
chunkConfigModalVisible.value = true; chunkConfigModalVisible.value = true;
}; };
@ -1005,6 +1014,7 @@ const showChunkConfigModal = () => {
const handleChunkConfigSubmit = () => { const handleChunkConfigSubmit = () => {
chunkParams.value.chunk_size = tempChunkParams.value.chunk_size; chunkParams.value.chunk_size = tempChunkParams.value.chunk_size;
chunkParams.value.chunk_overlap = tempChunkParams.value.chunk_overlap; chunkParams.value.chunk_overlap = tempChunkParams.value.chunk_overlap;
chunkParams.value.auto_indexing = tempChunkParams.value.auto_indexing;
chunkConfigModalVisible.value = false; chunkConfigModalVisible.value = false;
message.success('分块参数配置已更新'); message.success('分块参数配置已更新');
}; };

View File

@ -96,7 +96,7 @@
<ModelProvidersComponent /> <ModelProvidersComponent />
</div> </div>
<div class="setting" v-if="(state.windowWidth <= 520 || state.section ==='path') && userStore.isSuperAdmin"> <div class="setting" v-if="(state.windowWidth <= 520 || state.section ==='path') && userStore.isSuperAdmin">
<h3>本地模型配置</h3> <h3>本地模型配置将在 v0.2 版本移除对本地模型的支持</h3>
<p>如果是 Docker 启动务必确保在 docker-compose.dev.yaml 中添加了 volumes 映射</p> <p>如果是 Docker 启动务必确保在 docker-compose.dev.yaml 中添加了 volumes 映射</p>
<TableConfigComponent <TableConfigComponent
:config="configStore.config?.model_local_paths" :config="configStore.config?.model_local_paths"