支持使用URL添加到知识库；
This commit is contained in:
Wenjie Zhang
parent
46f252383f
commit
7449938775
@ -30,6 +30,7 @@ dependencies = [
|
||||
"rapidocr-onnxruntime>=1.4.4",
|
||||
"sentencepiece>=0.2.0",
|
||||
"tavily-python>=0.7.0",
|
||||
"unstructured>=0.17.2",
|
||||
"uvicorn[standard]>=0.34.2",
|
||||
"zhipuai>=2.1.5.20250421",
|
||||
]
|
||||
|
||||
@ -58,6 +58,12 @@ async def file_to_chunk(files: List[str] = Body(...), params: dict = Body(...),
|
||||
result = knowledge_base.file_to_chunk(files, params=params)
|
||||
return result
|
||||
|
||||
@data.post("/url-to-chunk")
|
||||
async def url_to_chunk(urls: List[str] = Body(...), params: dict = Body(...), current_user: User = Depends(get_admin_user)):
|
||||
logger.debug(f"Url to chunk: {urls}")
|
||||
result = knowledge_base.url_to_chunk(urls, params=params)
|
||||
return result
|
||||
|
||||
@data.post("/add-by-file")
|
||||
async def create_document_by_file(db_id: str = Body(...), files: List[str] = Body(...), current_user: User = Depends(get_admin_user)):
|
||||
logger.debug(f"Add document in {db_id} by file: {files}")
|
||||
|
||||
@ -258,6 +258,7 @@ class GraphDatabase:
|
||||
tx.run(query)
|
||||
|
||||
def query_node(self, entity_name, threshold=0.9, kgdb_name='neo4j', hops=2, max_entities=5, **kwargs):
|
||||
"""知识图谱查询节点的入口:"""
|
||||
# TODO 添加判断节点数量为 0 停止检索
|
||||
# 判断是否启动
|
||||
if not self.is_running():
|
||||
|
||||
@ -208,9 +208,75 @@ class KnowledgeBase:
|
||||
|
||||
return file_infos
|
||||
|
||||
def url_to_chunk(self, url, params=None):
|
||||
"""将url转换为分块，读取url的内容，并转换为分块"""
|
||||
raise NotImplementedError("Not implemented")
|
||||
def url_to_chunk(self, urls, params=None):
|
||||
"""将url转换为分块，读取url的内容，并转换为分块
|
||||
|
||||
Args:
|
||||
urls: list of urls
|
||||
params: params for chunking
|
||||
|
||||
Returns:
|
||||
dict: 包含分块信息的字典
|
||||
"""
|
||||
try:
|
||||
from langchain_community.document_loaders import UnstructuredURLLoader
|
||||
except ImportError:
|
||||
raise ImportError("请安装 langchain_community 和 unstructured 包：pip install langchain-community unstructured")
|
||||
|
||||
file_infos = {}
|
||||
|
||||
# 使用UnstructuredURLLoader加载URL内容
|
||||
loader = UnstructuredURLLoader(urls=urls, continue_on_failure=True)
|
||||
|
||||
for url_idx, url in enumerate(urls):
|
||||
file_id = "url_" + hashstr(url + str(time.time()))
|
||||
|
||||
try:
|
||||
# 加载单个URL内容
|
||||
single_loader = UnstructuredURLLoader(urls=[url], continue_on_failure=False)
|
||||
documents = single_loader.load()
|
||||
|
||||
# 将文档内容合并
|
||||
text_content = "\n\n".join([doc.page_content for doc in documents])
|
||||
|
||||
# 对内容进行分块
|
||||
nodes = chunk(text_content, params=params)
|
||||
|
||||
# 从URL中提取域名作为文件名
|
||||
from urllib.parse import urlparse, unquote
|
||||
parsed_url = urlparse(unquote(url))
|
||||
domain = parsed_url.netloc
|
||||
path = parsed_url.path
|
||||
filename = f"{domain}{path}"
|
||||
if filename.endswith('/'):
|
||||
filename = filename[:-1]
|
||||
if len(filename) > 100:
|
||||
filename = filename[:97] + "..."
|
||||
filename = filename.replace('/', '_')
|
||||
|
||||
file_infos[file_id] = {
|
||||
"file_id": file_id,
|
||||
"filename": filename,
|
||||
"path": url,
|
||||
"type": "url",
|
||||
"status": "waiting",
|
||||
"created_at": time.time(),
|
||||
"nodes": [node.dict() for node in nodes]
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"处理URL {url} 时出错: {e}")
|
||||
file_infos[file_id] = {
|
||||
"file_id": file_id,
|
||||
"filename": url[:100] + "..." if len(url) > 100 else url,
|
||||
"path": url,
|
||||
"type": "url",
|
||||
"status": "failed",
|
||||
"created_at": time.time(),
|
||||
"error": str(e),
|
||||
"nodes": []
|
||||
}
|
||||
|
||||
return file_infos
|
||||
|
||||
def add_chunks(self, db_id, file_chunks):
|
||||
"""添加分块"""
|
||||
|
||||
@ -180,7 +180,17 @@ export const knowledgeBaseApi = {
|
||||
getDocumentDetail: async (dbId, fileId) => {
|
||||
checkAdminPermission()
|
||||
return apiGet(`/api/data/document?db_id=${dbId}&file_id=${fileId}`, {}, true)
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* 将URL转换为分块
|
||||
* @param {Object} data - 分块参数
|
||||
* @returns {Promise} - 分块结果
|
||||
*/
|
||||
urlToChunk: async (data) => {
|
||||
checkAdminPermission()
|
||||
return apiPost('/api/data/url-to-chunk', data, {}, true)
|
||||
},
|
||||
}
|
||||
|
||||
// 图数据库管理API
|
||||
|
||||
@ -40,7 +40,7 @@ export async function apiRequest(url, options = {}, requiresAuth = false) {
|
||||
// 处理API返回的错误
|
||||
if (!response.ok) {
|
||||
// 尝试解析错误信息
|
||||
let errorMessage = `请求失败: ${response.status}`
|
||||
let errorMessage = `请求失败: ${response.status}, ${response.statusText}`
|
||||
let errorData = null
|
||||
|
||||
try {
|
||||
|
||||
64
web/src/components/LoadingComponent.vue
Normal file
64
web/src/components/LoadingComponent.vue
Normal file
@ -0,0 +1,64 @@
|
||||
<template>
|
||||
<div class="loading-container" v-if="visible">
|
||||
<div class="loading-content">
|
||||
<div class="loading-spinner"></div>
|
||||
<div class="loading-text">{{ text }}</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { defineProps } from 'vue';
|
||||
|
||||
const props = defineProps({
|
||||
visible: {
|
||||
type: Boolean,
|
||||
default: false
|
||||
},
|
||||
text: {
|
||||
type: String,
|
||||
default: '加载中...'
|
||||
}
|
||||
});
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.loading-container {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
background-color: rgba(255, 255, 255, 0.8);
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
z-index: 9999;
|
||||
}
|
||||
|
||||
.loading-content {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.loading-spinner {
|
||||
width: 50px;
|
||||
height: 50px;
|
||||
border: 5px solid #f3f3f3;
|
||||
border-top: 5px solid var(--main-color);
|
||||
border-radius: 50%;
|
||||
animation: spin 1s linear infinite;
|
||||
}
|
||||
|
||||
.loading-text {
|
||||
margin-top: 16px;
|
||||
font-size: 16px;
|
||||
color: var(--gray-900);
|
||||
}
|
||||
|
||||
@keyframes spin {
|
||||
0% { transform: rotate(0deg); }
|
||||
100% { transform: rotate(360deg); }
|
||||
}
|
||||
</style>
|
||||
@ -107,7 +107,19 @@
|
||||
</div>
|
||||
</div>
|
||||
<div class="upload-main">
|
||||
<div class="upload">
|
||||
<div class="source-selector">
|
||||
<a-radio-group v-model:value="uploadMode" button-style="solid" style="margin-bottom: 16px;">
|
||||
<a-radio-button value="file">
|
||||
<FileOutlined /> 上传文件
|
||||
</a-radio-button>
|
||||
<a-radio-button value="url">
|
||||
<LinkOutlined /> 输入网址
|
||||
</a-radio-button>
|
||||
</a-radio-group>
|
||||
</div>
|
||||
|
||||
<!-- 文件上传区域 -->
|
||||
<div class="upload" v-if="uploadMode === 'file'">
|
||||
<a-upload-dragger
|
||||
class="upload-dragger"
|
||||
v-model:fileList="fileList"
|
||||
@ -124,12 +136,30 @@
|
||||
</p>
|
||||
</a-upload-dragger>
|
||||
</div>
|
||||
|
||||
<!-- URL 输入区域 -->
|
||||
<div class="url-input" v-else>
|
||||
<a-form layout="vertical">
|
||||
<a-form-item label="网页链接 (每行一个URL)">
|
||||
<a-textarea
|
||||
v-model:value="urlList"
|
||||
placeholder="请输入网页链接，每行一个"
|
||||
:rows="6"
|
||||
:disabled="state.loading"
|
||||
/>
|
||||
</a-form-item>
|
||||
</a-form>
|
||||
<p class="url-hint">
|
||||
支持添加网页内容，系统会自动抓取网页文本并进行分块。请确保URL格式正确且可以公开访问。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="actions">
|
||||
<a-button
|
||||
type="primary"
|
||||
@click="chunkFiles"
|
||||
@click="chunkData"
|
||||
:loading="state.loading"
|
||||
:disabled="fileList.length === 0"
|
||||
:disabled="(uploadMode === 'file' && fileList.length === 0) || (uploadMode === 'url' && !urlList.trim())"
|
||||
style="margin: 0px 20px 20px 0;"
|
||||
>
|
||||
生成分块
|
||||
@ -299,6 +329,8 @@ import {
|
||||
CloudUploadOutlined,
|
||||
SearchOutlined,
|
||||
LoadingOutlined,
|
||||
FileOutlined,
|
||||
LinkOutlined,
|
||||
} from '@ant-design/icons-vue'
|
||||
|
||||
|
||||
@ -623,6 +655,39 @@ const chunkFiles = () => {
|
||||
})
|
||||
}
|
||||
|
||||
// 分块预览
|
||||
const chunkUrls = () => {
|
||||
// 分割并过滤URL列表
|
||||
const urls = urlList.value.split('\n')
|
||||
.map(url => url.trim())
|
||||
.filter(url => url.length > 0 && (url.startsWith('http://') || url.startsWith('https://')));
|
||||
|
||||
if (urls.length === 0) {
|
||||
message.error('请输入有效的网页链接（必须以http://或https://开头）');
|
||||
return;
|
||||
}
|
||||
|
||||
state.loading = true;
|
||||
|
||||
// 调用url-to-chunk接口获取分块信息
|
||||
knowledgeBaseApi.urlToChunk({
|
||||
urls: urls,
|
||||
params: chunkParams.value
|
||||
})
|
||||
.then(data => {
|
||||
console.log('URL分块信息:', data);
|
||||
chunkResults.value = Object.values(data);
|
||||
activeFileKeys.value = chunkResults.value.length > 0 ? [0] : []; // 默认展开第一个
|
||||
})
|
||||
.catch(error => {
|
||||
console.error(error);
|
||||
message.error(error.message || '处理URL失败');
|
||||
})
|
||||
.finally(() => {
|
||||
state.loading = false;
|
||||
});
|
||||
};
|
||||
|
||||
// 添加到数据库
|
||||
const addToDatabase = () => {
|
||||
if (chunkResults.value.length === 0) {
|
||||
@ -723,7 +788,16 @@ onUnmounted(() => {
|
||||
}
|
||||
})
|
||||
|
||||
const uploadMode = ref('file');
|
||||
const urlList = ref('');
|
||||
|
||||
const chunkData = () => {
|
||||
if (uploadMode.value === 'file') {
|
||||
chunkFiles();
|
||||
} else if (uploadMode.value === 'url') {
|
||||
chunkUrls();
|
||||
}
|
||||
}
|
||||
|
||||
</script>
|
||||
|
||||
@ -1181,6 +1255,29 @@ onUnmounted(() => {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.url-input {
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.url-input .ant-textarea {
|
||||
border-color: var(--main-light-3);
|
||||
background-color: #fff;
|
||||
font-family: monospace;
|
||||
resize: vertical;
|
||||
}
|
||||
|
||||
.url-input .ant-textarea:hover,
|
||||
.url-input .ant-textarea:focus {
|
||||
border-color: var(--main-color);
|
||||
}
|
||||
|
||||
.url-hint {
|
||||
font-size: 13px;
|
||||
color: var(--gray-600);
|
||||
margin-top: 5px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
</style>
|
||||
|
||||
<style lang="less">
|
||||
|
||||
Loading…
Reference in New Issue
Block a user