feat: 优化知识库和文件展示逻辑

- 更新了 KnowledgeDatabase 和 KnowledgeFile 的 to_dict 方法,支持选择性加载节点信息。
- 在 KnowledgeBase 中调整了数据库获取逻辑,确保在获取数据库时不加载文件节点。
- 在前端组件中添加了加载状态指示,改善了用户体验。
- 增强了数据库信息视图的交互性,支持批量索引和删除文件功能。
This commit is contained in:
Wenjie Zhang 2025-05-26 20:39:31 +08:00
parent 9d2604801d
commit b10edefe17
5 changed files with 245 additions and 89 deletions

View File

@ -21,7 +21,7 @@ class KnowledgeDatabase(Base):
# 关系
files = relationship("KnowledgeFile", back_populates="database", cascade="all, delete-orphan")
def to_dict(self):
def to_dict(self, with_nodes=True):
"""转换为字典格式确保meta_info映射为metadata"""
result = {
"id": self.id,
@ -30,16 +30,14 @@ class KnowledgeDatabase(Base):
"description": self.description,
"embed_model": self.embed_model,
"dimension": self.dimension,
"metadata": self.meta_info or {}, # 确保映射正确
"metadata": self.meta_info or {},
"created_at": self.created_at.isoformat() if self.created_at else None
}
# 添加文件信息
if self.files:
result["files"] = {file.file_id: file.to_dict() for file in self.files}
result["files"] = {file.file_id: file.to_dict(with_nodes=with_nodes) for file in self.files}
else:
result["files"] = {}
return result
class KnowledgeFile(Base):
@ -64,7 +62,7 @@ class KnowledgeFile(Base):
"""动态计算节点数量"""
return len(self.nodes) if self.nodes is not None else 0
def to_dict(self):
def to_dict(self, with_nodes=True):
"""转换为字典格式"""
result = {
"file_id": self.file_id,
@ -72,16 +70,11 @@ class KnowledgeFile(Base):
"path": self.path,
"type": self.file_type,
"status": self.status,
"node_count": self.computed_node_count, # 使用计算属性
"node_count": self.computed_node_count,
"created_at": self.created_at.timestamp() if self.created_at else time.time()
}
# 添加节点信息
if self.nodes:
result["nodes"] = [node.to_dict() for node in self.nodes]
else:
result["nodes"] = []
if with_nodes:
result["nodes"] = [node.to_dict() for node in self.nodes] if self.nodes else []
return result
class KnowledgeNode(Base):

View File

@ -7,6 +7,8 @@ from sqlalchemy.orm import joinedload
from pathlib import Path
import asyncio
import random
from functools import lru_cache
from diskcache import Cache
from pymilvus import MilvusClient, MilvusException
@ -22,6 +24,14 @@ class KnowledgeBase:
def __init__(self) -> None:
self.client = None
self.work_dir = os.path.join(config.save_dir, "data")
self.cache_dir = os.path.join(self.work_dir, ".cache")
os.makedirs(self.cache_dir, exist_ok=True)
# 初始化磁盘缓存
self.disk_cache = Cache(self.cache_dir)
# 配置缓存过期时间(秒)
self.cache_ttl = 300 # 5分钟
# Configuration
self.default_distance_threshold = 0.5
@ -38,7 +48,7 @@ class KnowledgeBase:
if hasattr(obj, 'to_dict'):
return obj.to_dict()
# For SQLAlchemy models not having a to_dict, or to customize
if isinstance(obj, (KnowledgeDatabase, KnowledgeFile, KnowledgeNode)):
if isinstance(obj, KnowledgeDatabase | KnowledgeFile | KnowledgeNode):
# Basic serialization, customize as needed
d = obj.__dict__.copy()
d.pop('_sa_instance_state', None)
@ -105,10 +115,8 @@ class KnowledgeBase:
def get_database_by_id(self, db_id):
"""根据ID获取知识库"""
with db_manager.get_session_context() as session:
db = session.query(KnowledgeDatabase).options(
joinedload(KnowledgeDatabase.files)
).filter_by(db_id=db_id).first()
return db.to_dict() if db else None # Assuming to_dict handles files and nodes
db = session.query(KnowledgeDatabase).options().filter_by(db_id=db_id).first()
return db.to_dict(with_nodes=False) if db else None # Assuming to_dict handles files and nodes
def create_database_record(self, db_id, name, description, embed_model=None, dimension=None, metadata=None):
"""在数据库中创建知识库记录"""
@ -716,27 +724,6 @@ class KnowledgeBase:
return self.search_by_vector(query_vectors[0], collection_name, limit)
def search_by_vector(self, vector, collection_name, limit=3):
# Default output fields for search, can be customized
# output_fields = ["text", "file_id", "hash", "start_char_idx", "end_char_idx"]
# valid_output_fields = output_fields # Default to all requested fields
# try:
# coll_info = self.client.describe_collection(collection_name)
# schema_fields = [field['name'] for field in coll_info.get('fields', [])]
# current_valid_fields = [f for f in output_fields if f in schema_fields]
# if current_valid_fields: # If any of the preferred fields are valid, use them
# valid_output_fields = current_valid_fields
# elif schema_fields: # Fallback: get all scalar fields if preferred are not found
# #This ensures we always try to get some data if the collection exists
# valid_output_fields = [f['name'] for f in coll_info.get('fields', []) if not f.get('is_primary', False) and coll_info.get('fields', [])[schema_fields.index(f)].get('type_name','').find('Vector') == -1 ]
# if not valid_output_fields and "text" in schema_fields: # Minimal fallback
# valid_output_fields = ["text"]
# elif not valid_output_fields and schema_fields: # If still nothing, take the first non-PK scalar field
# valid_output_fields = [schema_fields[0]] if schema_fields else []
# # If schema_fields is empty or no suitable fields, valid_output_fields might be empty; Milvus might error or return default.
# except Exception as e:
# logger.warning(f"Could not describe collection {collection_name} to validate output fields for search: {e}. Using default output_fields: {output_fields}.")
# # valid_output_fields remains as initially set (all requested output_fields)
res = self.client.search(
collection_name=collection_name,

View File

@ -8,6 +8,7 @@
</slot>
</div>
<div class="header-actions" v-if="$slots.actions">
<loading-outlined v-if="loading" />
<slot name="actions"></slot>
</div>
</div>
@ -15,6 +16,7 @@
</template>
<script setup>
import { LoadingOutlined } from '@ant-design/icons-vue';
const props = defineProps({
title: {
type: String,
@ -23,6 +25,10 @@ const props = defineProps({
description: {
type: String,
default: ''
},
loading: {
type: Boolean,
default: false
}
});
</script>

View File

@ -2,6 +2,7 @@
<div>
<HeaderComponent
:title="database.name || '数据库信息'"
:loading="state.databaseLoading"
>
<template #description>
<div class="database-info">
@ -42,9 +43,31 @@
<a-tab-pane key="files">
<template #tab><span><ReadOutlined />文件列表</span></template>
<div class="db-tab-container">
<div class="actions" style="display: flex; gap: 10px;">
<a-button type="primary" @click="handleShowAddFilesBlock" :loading="state.refrashing" :icon="h(PlusOutlined)">添加文件</a-button>
<a-button @click="handleRefresh" :loading="state.refrashing">刷新</a-button>
<div class="actions" style="display: flex; gap: 10px; justify-content: space-between;">
<div class="left-actions" style="display: flex; gap: 10px;">
<a-button type="primary" @click="handleShowAddFilesBlock" :loading="state.refrashing" :icon="h(PlusOutlined)">添加文件</a-button>
<a-button @click="handleRefresh" :loading="state.refrashing">刷新</a-button>
</div>
<div class="batch-actions" style="display: flex; gap: 10px;" v-if="selectedRowKeys.length > 0">
<span style="margin-right: 8px;">已选择 {{ selectedRowKeys.length }} </span>
<a-button
type="primary"
@click="handleBatchIndex"
:loading="state.batchIndexing"
:disabled="!hasSelectedPendingFiles"
>
批量索引
</a-button>
<a-button
type="primary"
danger
@click="handleBatchDelete"
:loading="state.batchDeleting"
:disabled="!canBatchDelete"
>
批量删除
</a-button>
</div>
</div>
<div class="upload-section" v-if="state.showAddFilesBlock">
<div class="upload-sidebar">
@ -92,7 +115,7 @@
v-model:fileList="fileList"
name="file"
:multiple="true"
:disabled="state.loading"
:disabled="state.chunkLoading"
:action="'/api/data/upload?db_id=' + databaseId"
:headers="getAuthHeaders()"
@change="handleFileUpload"
@ -113,7 +136,7 @@
v-model:value="urlList"
placeholder="请输入网页链接,每行一个"
:rows="6"
:disabled="state.loading"
:disabled="state.chunkLoading"
/>
</a-form-item>
</a-form>
@ -126,7 +149,7 @@
<a-button
type="primary"
@click="chunkData"
:loading="state.loading"
:loading="state.chunkLoading"
:disabled="(uploadMode === 'file' && fileList.length === 0) || (uploadMode === 'url' && !urlList.trim())"
style="margin: 0px 20px 20px 0;"
>
@ -135,7 +158,19 @@
</div>
</div>
</div>
<a-table :columns="columns" :data-source="Object.values(database.files || {})" row-key="file_id" class="my-table" size="small" bordered :pagination="pagination" >
<a-table
:columns="columns"
:data-source="Object.values(database.files || {})"
row-key="file_id"
class="my-table"
size="small"
bordered
:pagination="pagination"
:row-selection="{
selectedRowKeys: selectedRowKeys,
onChange: onSelectChange,
getCheckboxProps: getCheckboxProps
}">
<template #bodyCell="{ column, text, record }">
<template v-if="column.key === 'filename'">
<a-tooltip :title="record.file_id" placement="right">
@ -197,7 +232,7 @@
</div>
</template>
<template v-else>
<h2> {{ selectedFile?.lines?.length || 0 }} 个片段</h2>
<h3> {{ selectedFile?.lines?.length || 0 }} 个片段</h3>
<div class="file-detail-content">
<p v-for="line in selectedFile?.lines || []" :key="line.id" class="line-text">
{{ line.text }}
@ -368,7 +403,7 @@ const filteredResults = ref([])
const configStore = useConfigStore()
const state = reactive({
loading: false,
databaseLoading: false,
adding: false,
refrashing: false,
searchLoading: false,
@ -379,6 +414,9 @@ const state = reactive({
curPage: "files",
indexingFile: null,
showAddFilesBlock: false,
batchIndexing: false,
batchDeleting: false,
chunkLoading: false,
});
const meta = reactive({
@ -405,10 +443,13 @@ const use_rewrite_queryOptions = ref([
])
const pagination = ref({
pageSize: 30,
pageSize: 15,
current: 1,
total: 0,
total: computed(() => database.value?.files?.length || 0),
showSizeChanger: true,
onChange: (page, pageSize) => pagination.value.current = page,
showTotal: (total, range) => `${total}`,
onShowSizeChange: (current, pageSize) => pagination.value.pageSize = pageSize,
})
const filterQueryResults = () => {
@ -621,6 +662,7 @@ const getDatabaseInfo = () => {
return
}
state.lock = true
state.databaseLoading = true
return new Promise((resolve, reject) => {
knowledgeBaseApi.getDatabaseInfo(db_id)
.then(data => {
@ -634,6 +676,7 @@ const getDatabaseInfo = () => {
})
.finally(() => {
state.lock = false
state.databaseLoading = false
})
})
}
@ -693,7 +736,7 @@ const chunkFiles = () => {
return
}
state.loading = true
state.chunkLoading = true
knowledgeBaseApi.fileToChunk({
db_id: databaseId.value, // db_id
@ -717,7 +760,7 @@ const chunkFiles = () => {
message.error(error.message || '文件处理请求失败')
})
.finally(() => {
state.loading = false
state.chunkLoading = false
})
}
@ -732,7 +775,7 @@ const chunkUrls = () => {
return;
}
state.loading = true;
state.chunkLoading = true;
knowledgeBaseApi.urlToChunk({
db_id: databaseId.value, // db_id
@ -756,7 +799,7 @@ const chunkUrls = () => {
message.error(error.message || '处理URL请求失败');
})
.finally(() => {
state.loading = false;
state.chunkLoading = false;
});
};
@ -811,11 +854,11 @@ const addDocumentByFile = () => {
const columns = [
// { title: 'ID', dataIndex: 'file_id', key: 'file_id' },
{ title: '文件名', dataIndex: 'filename', key: 'filename' },
{ title: '上传时间', dataIndex: 'created_at', key: 'created_at' },
{ title: '状态', dataIndex: 'status', key: 'status' },
{ title: '类型', dataIndex: 'type', key: 'type' },
{ title: '操作', key: 'action', dataIndex: 'file_id' }
{ title: '文件名', dataIndex: 'filename', key: 'filename', ellipsis: true },
{ title: '上传时间', dataIndex: 'created_at', key: 'created_at', width: 150 },
{ title: '状态', dataIndex: 'status', key: 'status', width: 80 },
{ title: '类型', dataIndex: 'type', key: 'type', width: 80 },
{ title: '操作', key: 'action', dataIndex: 'file_id', width: 150 }
];
watch(() => route.params.database_id, (newId) => {
@ -929,37 +972,154 @@ const handleIndexFile = (fileId) => {
message.error('无效的文件ID');
return;
}
Modal.confirm({
title: '开始索引文件',
content: `确定要开始索引文件 ${fileId} 吗?该操作可能需要一些时间。`,
okText: '确认索引',
cancelText: '取消',
onOk: () => {
state.indexingFile = fileId; // Set loading state for this specific file
state.lock = true;
state.lock = true;
knowledgeBaseApi.indexFile({
db_id: databaseId.value,
file_id: fileId
})
.then(response => {
if (response.status === 'success') {
message.success(response.message || `文件 ${fileId} 已开始索引。`);
getDatabaseInfo(); // Refresh to update status
} else {
message.error(response.message || `文件 ${fileId} 索引启动失败。`);
}
})
.catch(error => {
console.error(`索引文件 ${fileId} 失败:`, error);
message.error(error.message || `索引文件 ${fileId} 时发生错误。`);
})
.finally(() => {
state.indexingFile = null; // Reset loading state for this file
state.lock = false;
});
};
//
const hasPendingFiles = computed(() => {
return Object.values(database.value.files || {}).some(file => file.status === 'pending_indexing');
});
//
const selectedRowKeys = ref([]);
//
const onSelectChange = (keys) => {
selectedRowKeys.value = keys;
};
//
const getCheckboxProps = (record) => ({
disabled: state.lock || record.status === 'processing' || record.status === 'waiting' || state.indexingFile === record.file_id,
});
//
const hasSelectedPendingFiles = computed(() => {
const files = database.value.files || {};
return selectedRowKeys.value.some(key => files[key]?.status === 'pending_indexing');
});
//
const canBatchDelete = computed(() => {
const files = database.value.files || {};
return selectedRowKeys.value.some(key => {
const file = files[key];
return !(state.lock || file.status === 'processing' || file.status === 'waiting' || state.indexingFile === file.file_id);
});
});
//
const handleBatchIndex = async () => {
if (!hasSelectedPendingFiles.value) {
message.info('没有需要索引的文件');
return;
}
state.batchIndexing = true;
const files = database.value.files || {};
const pendingFiles = selectedRowKeys.value
.filter(key => files[key]?.status === 'pending_indexing');
try {
const promises = pendingFiles.map(fileId =>
knowledgeBaseApi.indexFile({
db_id: databaseId.value,
file_id: fileId
})
.then(response => {
if (response.status === 'success') {
message.success(response.message || `文件 ${fileId} 已开始索引。`);
getDatabaseInfo(); // Refresh to update status
} else {
message.error(response.message || `文件 ${fileId} 索引启动失败。`);
);
const results = await Promise.allSettled(promises);
const succeeded = results.filter(r => r.status === 'fulfilled' && r.value.status === 'success').length;
const failed = results.filter(r => r.status === 'rejected' || (r.status === 'fulfilled' && r.value.status !== 'success')).length;
if (succeeded > 0) {
message.success(`成功提交 ${succeeded} 个文件进行索引`);
}
if (failed > 0) {
message.error(`${failed} 个文件提交索引失败`);
}
selectedRowKeys.value = []; //
getDatabaseInfo(); //
} catch (error) {
console.error('批量索引出错:', error);
message.error('批量索引过程中发生错误');
} finally {
state.batchIndexing = false;
}
};
//
const handleBatchDelete = () => {
if (!canBatchDelete.value) {
message.info('没有可删除的文件');
return;
}
const files = database.value.files || {};
const fileCount = selectedRowKeys.value.length;
Modal.confirm({
title: '批量删除文件',
content: `确定要删除选中的 ${fileCount} 个文件吗?`,
okText: '确认',
cancelText: '取消',
onOk: async () => {
state.batchDeleting = true;
state.lock = true;
try {
const promises = selectedRowKeys.value
.filter(fileId => {
const file = files[fileId];
return !(state.lock || file.status === 'processing' || file.status === 'waiting' || state.indexingFile === file.file_id);
})
.map(fileId =>
knowledgeBaseApi.deleteFile(databaseId.value, fileId)
);
const results = await Promise.allSettled(promises);
const succeeded = results.filter(r => r.status === 'fulfilled').length;
const failed = results.filter(r => r.status === 'rejected').length;
if (succeeded > 0) {
message.success(`成功删除 ${succeeded} 个文件`);
}
})
.catch(error => {
console.error(`索引文件 ${fileId} 失败:`, error);
message.error(error.message || `索引文件 ${fileId} 时发生错误。`);
})
.finally(() => {
state.indexingFile = null; // Reset loading state for this file
if (failed > 0) {
message.error(`${failed} 个文件删除失败`);
}
selectedRowKeys.value = []; //
getDatabaseInfo(); //
} catch (error) {
console.error('批量删除出错:', error);
message.error('批量删除过程中发生错误');
} finally {
state.batchDeleting = false;
state.lock = false;
});
},
onCancel: () => {
console.log('取消索引');
}
},
});
};
@ -1259,6 +1419,7 @@ const handleIndexFile = (fileId) => {
button.main-btn {
font-weight: bold;
font-size: 14px;
color: var(--gray-800);
&:hover {
cursor: pointer;
color: var(--main-color);
@ -1281,13 +1442,14 @@ const handleIndexFile = (fileId) => {
.file-detail-content {
max-height: 60vh;
overflow-y: auto;
padding: 0 10px;
// padding: 0 10px;
}
.custom-class .line-text {
padding: 10px;
border-radius: 4px;
margin: 8px 0;
background-color: var(--gray-200);
&:hover {
background-color: var(--main-light-4);
@ -1299,7 +1461,8 @@ const handleIndexFile = (fileId) => {
gap: 20px;
.upload-sidebar {
width: 280px;
min-width: 280px;
height: fit-content;
padding: 20px;
background-color: var(--main-light-6);
border-radius: 8px;

View File

@ -1,6 +1,6 @@
<template>
<div class="database-container layout-container" v-if="configStore.config.enable_knowledge_base">
<HeaderComponent title="文档知识库">
<HeaderComponent title="文档知识库" :loading="state.loading">
<template #actions>
<a-button type="primary" @click="newDatabase.open=true">新建知识库</a-button>
</template>
@ -105,6 +105,10 @@ const userStore = useUserStore()
const indicator = h(LoadingOutlined, {spin: true});
const configStore = useConfigStore()
const state = reactive({
loading: false,
})
const newDatabase = reactive({
name: '',
description: '',
@ -113,17 +117,20 @@ const newDatabase = reactive({
})
const loadDatabases = () => {
state.loading = true
// loadGraph()
knowledgeBaseApi.getDatabases()
.then(data => {
console.log(data)
databases.value = data.databases
state.loading = false
})
.catch(error => {
console.error('加载数据库列表失败:', error);
if (error.message.includes('权限')) {
message.error('需要管理员权限访问知识库')
}
}
state.loading = false
})
}