feat: 添加自动索引功能,优化文件处理流程
- 在知识库中新增自动索引参数,支持在文件处理后自动触发索引。 - 更新前端界面,允许用户配置自动索引选项,并在处理结果中反馈索引状态。 - 调整定时器刷新间隔,提高数据库信息更新的实时性。
This commit is contained in:
parent
5aebc248c3
commit
4afbb42f96
@ -460,6 +460,9 @@ class KnowledgeBase:
|
|||||||
self.update_file_status(file_id, "pending_indexing")
|
self.update_file_status(file_id, "pending_indexing")
|
||||||
file_record['status'] = "pending_indexing" # Ensure status is up-to-date
|
file_record['status'] = "pending_indexing" # Ensure status is up-to-date
|
||||||
|
|
||||||
|
if params.get("auto_indexing", False):
|
||||||
|
await self.trigger_file_indexing(db_id, file_id)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"处理文件 {file_path} 失败,无法保存待索引块: {e}, {traceback.format_exc()}")
|
logger.error(f"处理文件 {file_path} 失败,无法保存待索引块: {e}, {traceback.format_exc()}")
|
||||||
self.update_file_status(file_id, "failed") # Mark file as failed
|
self.update_file_status(file_id, "failed") # Mark file as failed
|
||||||
@ -510,6 +513,9 @@ class KnowledgeBase:
|
|||||||
self.update_file_status(file_id, "pending_indexing")
|
self.update_file_status(file_id, "pending_indexing")
|
||||||
file_record['status'] = "pending_indexing"
|
file_record['status'] = "pending_indexing"
|
||||||
|
|
||||||
|
if params.get("auto_indexing", False):
|
||||||
|
await self.trigger_file_indexing(db_id, file_id)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"处理URL {url} 失败,无法保存待索引块: {e}, {traceback.format_exc()}")
|
logger.error(f"处理URL {url} 失败,无法保存待索引块: {e}, {traceback.format_exc()}")
|
||||||
self.update_file_status(file_id, "failed")
|
self.update_file_status(file_id, "failed")
|
||||||
|
|||||||
@ -159,7 +159,7 @@ class OtherEmbedding(BaseEmbeddingModel):
|
|||||||
self.embed_model_fullname = config.embed_model
|
self.embed_model_fullname = config.embed_model
|
||||||
self.dimension = self.info.get("dimension", None)
|
self.dimension = self.info.get("dimension", None)
|
||||||
self.model = self.info["name"]
|
self.model = self.info["name"]
|
||||||
self.api_key = os.getenv(self.info["api_key"], None)
|
self.api_key = os.getenv(self.info["api_key"], self.info["api_key"])
|
||||||
self.url = get_docker_safe_url(self.info["url"])
|
self.url = get_docker_safe_url(self.info["url"])
|
||||||
assert self.url and self.model, f"URL and model are required. Cur embed model: {config.embed_model}"
|
assert self.url and self.model, f"URL and model are required. Cur embed model: {config.embed_model}"
|
||||||
self.headers = {
|
self.headers = {
|
||||||
@ -190,6 +190,7 @@ def get_embedding_model():
|
|||||||
assert config.embed_model in support_embed_models, f"Unsupported embed model: {config.embed_model}, only support {support_embed_models}"
|
assert config.embed_model in support_embed_models, f"Unsupported embed model: {config.embed_model}, only support {support_embed_models}"
|
||||||
logger.debug(f"Loading embedding model {config.embed_model}")
|
logger.debug(f"Loading embedding model {config.embed_model}")
|
||||||
if provider == "local":
|
if provider == "local":
|
||||||
|
logger.warning("[DEPRECATED] Local embedding model will be removed in v0.2, please use other embedding models")
|
||||||
model = LocalEmbeddingModel()
|
model = LocalEmbeddingModel()
|
||||||
|
|
||||||
elif provider == "zhipu":
|
elif provider == "zhipu":
|
||||||
|
|||||||
@ -22,7 +22,7 @@ class LocalReranker(FlagReranker):
|
|||||||
def sigmoid(x):
|
def sigmoid(x):
|
||||||
return 1 / (1 + np.exp(-x))
|
return 1 / (1 + np.exp(-x))
|
||||||
|
|
||||||
class SiliconFlowReranker:
|
class OnlineRerank:
|
||||||
def __init__(self, **kwargs):
|
def __init__(self, **kwargs):
|
||||||
model_info = config.reranker_names[config.reranker]
|
model_info = config.reranker_names[config.reranker]
|
||||||
self.url = get_docker_safe_url(model_info["url"])
|
self.url = get_docker_safe_url(model_info["url"])
|
||||||
@ -64,9 +64,10 @@ def get_reranker():
|
|||||||
assert config.reranker in support_rerankers, f"Unsupported Reranker: {config.reranker}, only support {support_rerankers}"
|
assert config.reranker in support_rerankers, f"Unsupported Reranker: {config.reranker}, only support {support_rerankers}"
|
||||||
provider, model_name = config.reranker.split('/', 1)
|
provider, model_name = config.reranker.split('/', 1)
|
||||||
if provider == "local":
|
if provider == "local":
|
||||||
|
logger.warning("[DEPRECATED] Local reranker will be removed in v0.2, please use other reranker")
|
||||||
return LocalReranker()
|
return LocalReranker()
|
||||||
elif provider == "siliconflow":
|
elif provider == "siliconflow":
|
||||||
return SiliconFlowReranker()
|
return OnlineRerank()
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"Unsupported Reranker: {config.reranker}, only support {config.reranker_names.keys()}")
|
raise ValueError(f"Unsupported Reranker: {config.reranker}, only support {config.reranker_names.keys()}")
|
||||||
|
|
||||||
|
|||||||
@ -111,11 +111,6 @@ MODEL_NAMES:
|
|||||||
|
|
||||||
|
|
||||||
EMBED_MODEL_INFO:
|
EMBED_MODEL_INFO:
|
||||||
local/BAAI/bge-m3:
|
|
||||||
name: BAAI/bge-m3
|
|
||||||
dimension: 1024
|
|
||||||
# local_path: /models/BAAI/bge-m3,也可以在这里配置
|
|
||||||
|
|
||||||
zhipu/zhipu-embedding-2:
|
zhipu/zhipu-embedding-2:
|
||||||
name: embedding-2
|
name: embedding-2
|
||||||
dimension: 1024
|
dimension: 1024
|
||||||
@ -131,25 +126,23 @@ EMBED_MODEL_INFO:
|
|||||||
api_key: SILICONFLOW_API_KEY
|
api_key: SILICONFLOW_API_KEY
|
||||||
|
|
||||||
vllm/Qwen/Qwen3-Embedding-0.6B:
|
vllm/Qwen/Qwen3-Embedding-0.6B:
|
||||||
name: qwen3-embedding-0.6b
|
name: Qwen3-Embedding-0.6B
|
||||||
dimension: 1024
|
dimension: 1024
|
||||||
url: http://172.19.13.6:8081/v1/embeddings
|
url: http://localhost:8081/v1/embeddings
|
||||||
api_key: no_api_key
|
api_key: no_api_key
|
||||||
|
|
||||||
ollama/nomic-embed-text:
|
ollama/nomic-embed-text:
|
||||||
name: nomic-embed-text
|
name: nomic-embed-text
|
||||||
|
url: http://localhost:11434/api/embed
|
||||||
dimension: 768
|
dimension: 768
|
||||||
|
|
||||||
ollama/bge-m3:
|
ollama/bge-m3:
|
||||||
name: bge-m3
|
name: bge-m3
|
||||||
|
url: http://localhost:11434/api/embed
|
||||||
dimension: 1024
|
dimension: 1024
|
||||||
|
|
||||||
RERANKER_LIST:
|
RERANKER_LIST:
|
||||||
|
|
||||||
local/BAAI/bge-reranker-v2-m3:
|
|
||||||
name: BAAI/bge-reranker-v2-m3
|
|
||||||
# local_path: /models/BAAI/bge-m3,也可以在这里配置
|
|
||||||
|
|
||||||
siliconflow/BAAI/bge-reranker-v2-m3:
|
siliconflow/BAAI/bge-reranker-v2-m3:
|
||||||
name: BAAI/bge-reranker-v2-m3
|
name: BAAI/bge-reranker-v2-m3
|
||||||
url: https://api.siliconflow.cn/v1/rerank
|
url: https://api.siliconflow.cn/v1/rerank
|
||||||
@ -157,5 +150,5 @@ RERANKER_LIST:
|
|||||||
|
|
||||||
vllm/Qwen/Qwen3-Reranker-0.6B:
|
vllm/Qwen/Qwen3-Reranker-0.6B:
|
||||||
name: Qwen/Qwen3-Reranker-0.6B
|
name: Qwen/Qwen3-Reranker-0.6B
|
||||||
url: http://172.19.13.6:8081/v1/rerank
|
url: http://localhost:8081/v1/rerank
|
||||||
api_key: no_api_key
|
api_key: no_api_key
|
||||||
|
|||||||
@ -65,6 +65,10 @@
|
|||||||
<a-input-number v-model:value="tempChunkParams.chunk_overlap" :min="0" :max="1000" style="width: 100%;" />
|
<a-input-number v-model:value="tempChunkParams.chunk_overlap" :min="0" :max="1000" style="width: 100%;" />
|
||||||
<p class="param-description">相邻文本片段间的重叠字符数</p>
|
<p class="param-description">相邻文本片段间的重叠字符数</p>
|
||||||
</a-form-item>
|
</a-form-item>
|
||||||
|
<a-form-item label="自动索引" name="auto_indexing">
|
||||||
|
<a-switch v-model:checked="tempChunkParams.auto_indexing" />
|
||||||
|
<p class="param-description">分块完成后自动开始索引,生成向量并写入数据库</p>
|
||||||
|
</a-form-item>
|
||||||
</a-form>
|
</a-form>
|
||||||
</div>
|
</div>
|
||||||
</a-modal>
|
</a-modal>
|
||||||
@ -95,7 +99,7 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="config-controls">
|
<div class="config-controls">
|
||||||
<a-button type="dashed" @click="showChunkConfigModal">
|
<a-button type="dashed" @click="showChunkConfigModal">
|
||||||
<SettingOutlined /> 分块参数 ({{ chunkParams.chunk_size }}/{{ chunkParams.chunk_overlap }})
|
<SettingOutlined /> 分块参数 ({{ chunkParams.chunk_size }}/{{ chunkParams.chunk_overlap }}{{ chunkParams.auto_indexing ? '/自动索引' : '' }})
|
||||||
</a-button>
|
</a-button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@ -781,6 +785,7 @@ const chunkParams = ref({
|
|||||||
chunk_size: 1000,
|
chunk_size: 1000,
|
||||||
chunk_overlap: 200,
|
chunk_overlap: 200,
|
||||||
enable_ocr: 'disable',
|
enable_ocr: 'disable',
|
||||||
|
auto_indexing: false,
|
||||||
})
|
})
|
||||||
|
|
||||||
const chunkResults = ref([]);
|
const chunkResults = ref([]);
|
||||||
@ -812,7 +817,8 @@ const chunkFiles = () => {
|
|||||||
.then(data => {
|
.then(data => {
|
||||||
console.log('文件处理结果:', data)
|
console.log('文件处理结果:', data)
|
||||||
if (data.status === 'success') {
|
if (data.status === 'success') {
|
||||||
message.info(data.message || '文件已提交处理,请稍后在列表刷新查看状态');
|
const autoIndexingInfo = chunkParams.value.auto_indexing ? ',自动索引已启动' : ',请手动点击索引按钮完成索引';
|
||||||
|
message.info(data.message + autoIndexingInfo || '文件已提交处理,请稍后在列表刷新查看状态');
|
||||||
fileList.value = []; // 清空已上传文件列表
|
fileList.value = []; // 清空已上传文件列表
|
||||||
addFilesModalVisible.value = false; // 关闭弹窗
|
addFilesModalVisible.value = false; // 关闭弹窗
|
||||||
getDatabaseInfo(); // 刷新数据库信息以显示新文件及其状态
|
getDatabaseInfo(); // 刷新数据库信息以显示新文件及其状态
|
||||||
@ -850,7 +856,8 @@ const chunkUrls = () => {
|
|||||||
.then(data => {
|
.then(data => {
|
||||||
console.log('URL处理结果:', data);
|
console.log('URL处理结果:', data);
|
||||||
if (data.status === 'success') {
|
if (data.status === 'success') {
|
||||||
message.success(data.message || 'URL已提交处理,请稍后在列表刷新查看状态');
|
const autoIndexingInfo = chunkParams.value.auto_indexing ? ',自动索引已启动' : ',请手动点击索引按钮完成索引';
|
||||||
|
message.success(data.message + autoIndexingInfo || 'URL已提交处理,请稍后在列表刷新查看状态');
|
||||||
urlList.value = ''; // 清空URL输入
|
urlList.value = ''; // 清空URL输入
|
||||||
addFilesModalVisible.value = false; // 关闭弹窗
|
addFilesModalVisible.value = false; // 关闭弹窗
|
||||||
getDatabaseInfo(); // 刷新数据库信息以显示新文件及其状态
|
getDatabaseInfo(); // 刷新数据库信息以显示新文件及其状态
|
||||||
@ -907,7 +914,7 @@ onMounted(() => {
|
|||||||
getDatabaseInfo();
|
getDatabaseInfo();
|
||||||
state.refreshInterval = setInterval(() => {
|
state.refreshInterval = setInterval(() => {
|
||||||
getDatabaseInfo();
|
getDatabaseInfo();
|
||||||
}, 10000);
|
}, 1000);
|
||||||
})
|
})
|
||||||
|
|
||||||
// 添加 onUnmounted 钩子,在组件卸载时清除定时器
|
// 添加 onUnmounted 钩子,在组件卸载时清除定时器
|
||||||
@ -951,6 +958,7 @@ const chunkConfigModalVisible = ref(false);
|
|||||||
const tempChunkParams = ref({
|
const tempChunkParams = ref({
|
||||||
chunk_size: 1000,
|
chunk_size: 1000,
|
||||||
chunk_overlap: 200,
|
chunk_overlap: 200,
|
||||||
|
auto_indexing: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
// 添加文件弹窗
|
// 添加文件弹窗
|
||||||
@ -997,6 +1005,7 @@ const showChunkConfigModal = () => {
|
|||||||
tempChunkParams.value = {
|
tempChunkParams.value = {
|
||||||
chunk_size: chunkParams.value.chunk_size,
|
chunk_size: chunkParams.value.chunk_size,
|
||||||
chunk_overlap: chunkParams.value.chunk_overlap,
|
chunk_overlap: chunkParams.value.chunk_overlap,
|
||||||
|
auto_indexing: chunkParams.value.auto_indexing,
|
||||||
};
|
};
|
||||||
chunkConfigModalVisible.value = true;
|
chunkConfigModalVisible.value = true;
|
||||||
};
|
};
|
||||||
@ -1005,6 +1014,7 @@ const showChunkConfigModal = () => {
|
|||||||
const handleChunkConfigSubmit = () => {
|
const handleChunkConfigSubmit = () => {
|
||||||
chunkParams.value.chunk_size = tempChunkParams.value.chunk_size;
|
chunkParams.value.chunk_size = tempChunkParams.value.chunk_size;
|
||||||
chunkParams.value.chunk_overlap = tempChunkParams.value.chunk_overlap;
|
chunkParams.value.chunk_overlap = tempChunkParams.value.chunk_overlap;
|
||||||
|
chunkParams.value.auto_indexing = tempChunkParams.value.auto_indexing;
|
||||||
chunkConfigModalVisible.value = false;
|
chunkConfigModalVisible.value = false;
|
||||||
message.success('分块参数配置已更新');
|
message.success('分块参数配置已更新');
|
||||||
};
|
};
|
||||||
|
|||||||
@ -96,7 +96,7 @@
|
|||||||
<ModelProvidersComponent />
|
<ModelProvidersComponent />
|
||||||
</div>
|
</div>
|
||||||
<div class="setting" v-if="(state.windowWidth <= 520 || state.section ==='path') && userStore.isSuperAdmin">
|
<div class="setting" v-if="(state.windowWidth <= 520 || state.section ==='path') && userStore.isSuperAdmin">
|
||||||
<h3>本地模型配置</h3>
|
<h3>本地模型配置(将在 v0.2 版本移除对本地模型的支持)</h3>
|
||||||
<p>如果是 Docker 启动,务必确保在 docker-compose.dev.yaml 中添加了 volumes 映射。</p>
|
<p>如果是 Docker 启动,务必确保在 docker-compose.dev.yaml 中添加了 volumes 映射。</p>
|
||||||
<TableConfigComponent
|
<TableConfigComponent
|
||||||
:config="configStore.config?.model_local_paths"
|
:config="configStore.config?.model_local_paths"
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user