feat: 添加自动索引功能,优化文件处理流程
- 在知识库中新增自动索引参数,支持在文件处理后自动触发索引。 - 更新前端界面,允许用户配置自动索引选项,并在处理结果中反馈索引状态。 - 调整定时器刷新间隔,提高数据库信息更新的实时性。
This commit is contained in:
parent
5aebc248c3
commit
4afbb42f96
@ -460,6 +460,9 @@ class KnowledgeBase:
|
||||
self.update_file_status(file_id, "pending_indexing")
|
||||
file_record['status'] = "pending_indexing" # Ensure status is up-to-date
|
||||
|
||||
if params.get("auto_indexing", False):
|
||||
await self.trigger_file_indexing(db_id, file_id)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理文件 {file_path} 失败,无法保存待索引块: {e}, {traceback.format_exc()}")
|
||||
self.update_file_status(file_id, "failed") # Mark file as failed
|
||||
@ -510,6 +513,9 @@ class KnowledgeBase:
|
||||
self.update_file_status(file_id, "pending_indexing")
|
||||
file_record['status'] = "pending_indexing"
|
||||
|
||||
if params.get("auto_indexing", False):
|
||||
await self.trigger_file_indexing(db_id, file_id)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理URL {url} 失败,无法保存待索引块: {e}, {traceback.format_exc()}")
|
||||
self.update_file_status(file_id, "failed")
|
||||
|
||||
@ -159,7 +159,7 @@ class OtherEmbedding(BaseEmbeddingModel):
|
||||
self.embed_model_fullname = config.embed_model
|
||||
self.dimension = self.info.get("dimension", None)
|
||||
self.model = self.info["name"]
|
||||
self.api_key = os.getenv(self.info["api_key"], None)
|
||||
self.api_key = os.getenv(self.info["api_key"], self.info["api_key"])
|
||||
self.url = get_docker_safe_url(self.info["url"])
|
||||
assert self.url and self.model, f"URL and model are required. Cur embed model: {config.embed_model}"
|
||||
self.headers = {
|
||||
@ -190,6 +190,7 @@ def get_embedding_model():
|
||||
assert config.embed_model in support_embed_models, f"Unsupported embed model: {config.embed_model}, only support {support_embed_models}"
|
||||
logger.debug(f"Loading embedding model {config.embed_model}")
|
||||
if provider == "local":
|
||||
logger.warning("[DEPRECATED] Local embedding model will be removed in v0.2, please use other embedding models")
|
||||
model = LocalEmbeddingModel()
|
||||
|
||||
elif provider == "zhipu":
|
||||
|
||||
@ -22,7 +22,7 @@ class LocalReranker(FlagReranker):
|
||||
def sigmoid(x):
|
||||
return 1 / (1 + np.exp(-x))
|
||||
|
||||
class SiliconFlowReranker:
|
||||
class OnlineRerank:
|
||||
def __init__(self, **kwargs):
|
||||
model_info = config.reranker_names[config.reranker]
|
||||
self.url = get_docker_safe_url(model_info["url"])
|
||||
@ -64,9 +64,10 @@ def get_reranker():
|
||||
assert config.reranker in support_rerankers, f"Unsupported Reranker: {config.reranker}, only support {support_rerankers}"
|
||||
provider, model_name = config.reranker.split('/', 1)
|
||||
if provider == "local":
|
||||
logger.warning("[DEPRECATED] Local reranker will be removed in v0.2, please use other reranker")
|
||||
return LocalReranker()
|
||||
elif provider == "siliconflow":
|
||||
return SiliconFlowReranker()
|
||||
return OnlineRerank()
|
||||
else:
|
||||
raise ValueError(f"Unsupported Reranker: {config.reranker}, only support {config.reranker_names.keys()}")
|
||||
|
||||
|
||||
@ -111,11 +111,6 @@ MODEL_NAMES:
|
||||
|
||||
|
||||
EMBED_MODEL_INFO:
|
||||
local/BAAI/bge-m3:
|
||||
name: BAAI/bge-m3
|
||||
dimension: 1024
|
||||
# local_path: /models/BAAI/bge-m3,也可以在这里配置
|
||||
|
||||
zhipu/zhipu-embedding-2:
|
||||
name: embedding-2
|
||||
dimension: 1024
|
||||
@ -131,25 +126,23 @@ EMBED_MODEL_INFO:
|
||||
api_key: SILICONFLOW_API_KEY
|
||||
|
||||
vllm/Qwen/Qwen3-Embedding-0.6B:
|
||||
name: qwen3-embedding-0.6b
|
||||
name: Qwen3-Embedding-0.6B
|
||||
dimension: 1024
|
||||
url: http://172.19.13.6:8081/v1/embeddings
|
||||
url: http://localhost:8081/v1/embeddings
|
||||
api_key: no_api_key
|
||||
|
||||
ollama/nomic-embed-text:
|
||||
name: nomic-embed-text
|
||||
url: http://localhost:11434/api/embed
|
||||
dimension: 768
|
||||
|
||||
ollama/bge-m3:
|
||||
name: bge-m3
|
||||
url: http://localhost:11434/api/embed
|
||||
dimension: 1024
|
||||
|
||||
RERANKER_LIST:
|
||||
|
||||
local/BAAI/bge-reranker-v2-m3:
|
||||
name: BAAI/bge-reranker-v2-m3
|
||||
# local_path: /models/BAAI/bge-m3,也可以在这里配置
|
||||
|
||||
siliconflow/BAAI/bge-reranker-v2-m3:
|
||||
name: BAAI/bge-reranker-v2-m3
|
||||
url: https://api.siliconflow.cn/v1/rerank
|
||||
@ -157,5 +150,5 @@ RERANKER_LIST:
|
||||
|
||||
vllm/Qwen/Qwen3-Reranker-0.6B:
|
||||
name: Qwen/Qwen3-Reranker-0.6B
|
||||
url: http://172.19.13.6:8081/v1/rerank
|
||||
url: http://localhost:8081/v1/rerank
|
||||
api_key: no_api_key
|
||||
|
||||
@ -65,6 +65,10 @@
|
||||
<a-input-number v-model:value="tempChunkParams.chunk_overlap" :min="0" :max="1000" style="width: 100%;" />
|
||||
<p class="param-description">相邻文本片段间的重叠字符数</p>
|
||||
</a-form-item>
|
||||
<a-form-item label="自动索引" name="auto_indexing">
|
||||
<a-switch v-model:checked="tempChunkParams.auto_indexing" />
|
||||
<p class="param-description">分块完成后自动开始索引,生成向量并写入数据库</p>
|
||||
</a-form-item>
|
||||
</a-form>
|
||||
</div>
|
||||
</a-modal>
|
||||
@ -95,7 +99,7 @@
|
||||
</div>
|
||||
<div class="config-controls">
|
||||
<a-button type="dashed" @click="showChunkConfigModal">
|
||||
<SettingOutlined /> 分块参数 ({{ chunkParams.chunk_size }}/{{ chunkParams.chunk_overlap }})
|
||||
<SettingOutlined /> 分块参数 ({{ chunkParams.chunk_size }}/{{ chunkParams.chunk_overlap }}{{ chunkParams.auto_indexing ? '/自动索引' : '' }})
|
||||
</a-button>
|
||||
</div>
|
||||
</div>
|
||||
@ -781,6 +785,7 @@ const chunkParams = ref({
|
||||
chunk_size: 1000,
|
||||
chunk_overlap: 200,
|
||||
enable_ocr: 'disable',
|
||||
auto_indexing: false,
|
||||
})
|
||||
|
||||
const chunkResults = ref([]);
|
||||
@ -812,7 +817,8 @@ const chunkFiles = () => {
|
||||
.then(data => {
|
||||
console.log('文件处理结果:', data)
|
||||
if (data.status === 'success') {
|
||||
message.info(data.message || '文件已提交处理,请稍后在列表刷新查看状态');
|
||||
const autoIndexingInfo = chunkParams.value.auto_indexing ? ',自动索引已启动' : ',请手动点击索引按钮完成索引';
|
||||
message.info(data.message + autoIndexingInfo || '文件已提交处理,请稍后在列表刷新查看状态');
|
||||
fileList.value = []; // 清空已上传文件列表
|
||||
addFilesModalVisible.value = false; // 关闭弹窗
|
||||
getDatabaseInfo(); // 刷新数据库信息以显示新文件及其状态
|
||||
@ -850,7 +856,8 @@ const chunkUrls = () => {
|
||||
.then(data => {
|
||||
console.log('URL处理结果:', data);
|
||||
if (data.status === 'success') {
|
||||
message.success(data.message || 'URL已提交处理,请稍后在列表刷新查看状态');
|
||||
const autoIndexingInfo = chunkParams.value.auto_indexing ? ',自动索引已启动' : ',请手动点击索引按钮完成索引';
|
||||
message.success(data.message + autoIndexingInfo || 'URL已提交处理,请稍后在列表刷新查看状态');
|
||||
urlList.value = ''; // 清空URL输入
|
||||
addFilesModalVisible.value = false; // 关闭弹窗
|
||||
getDatabaseInfo(); // 刷新数据库信息以显示新文件及其状态
|
||||
@ -907,7 +914,7 @@ onMounted(() => {
|
||||
getDatabaseInfo();
|
||||
state.refreshInterval = setInterval(() => {
|
||||
getDatabaseInfo();
|
||||
}, 10000);
|
||||
}, 1000);
|
||||
})
|
||||
|
||||
// 添加 onUnmounted 钩子,在组件卸载时清除定时器
|
||||
@ -951,6 +958,7 @@ const chunkConfigModalVisible = ref(false);
|
||||
const tempChunkParams = ref({
|
||||
chunk_size: 1000,
|
||||
chunk_overlap: 200,
|
||||
auto_indexing: false,
|
||||
});
|
||||
|
||||
// 添加文件弹窗
|
||||
@ -997,6 +1005,7 @@ const showChunkConfigModal = () => {
|
||||
tempChunkParams.value = {
|
||||
chunk_size: chunkParams.value.chunk_size,
|
||||
chunk_overlap: chunkParams.value.chunk_overlap,
|
||||
auto_indexing: chunkParams.value.auto_indexing,
|
||||
};
|
||||
chunkConfigModalVisible.value = true;
|
||||
};
|
||||
@ -1005,6 +1014,7 @@ const showChunkConfigModal = () => {
|
||||
const handleChunkConfigSubmit = () => {
|
||||
chunkParams.value.chunk_size = tempChunkParams.value.chunk_size;
|
||||
chunkParams.value.chunk_overlap = tempChunkParams.value.chunk_overlap;
|
||||
chunkParams.value.auto_indexing = tempChunkParams.value.auto_indexing;
|
||||
chunkConfigModalVisible.value = false;
|
||||
message.success('分块参数配置已更新');
|
||||
};
|
||||
|
||||
@ -96,7 +96,7 @@
|
||||
<ModelProvidersComponent />
|
||||
</div>
|
||||
<div class="setting" v-if="(state.windowWidth <= 520 || state.section ==='path') && userStore.isSuperAdmin">
|
||||
<h3>本地模型配置</h3>
|
||||
<h3>本地模型配置(将在 v0.2 版本移除对本地模型的支持)</h3>
|
||||
<p>如果是 Docker 启动,务必确保在 docker-compose.dev.yaml 中添加了 volumes 映射。</p>
|
||||
<TableConfigComponent
|
||||
:config="configStore.config?.model_local_paths"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user