This commit is contained in:
Wenjie Zhang 2024-09-06 12:54:17 +08:00
parent 6fd7bf797d
commit 0d455bcc3d
14 changed files with 359 additions and 125 deletions

8
.gitignore vendored
View File

@ -25,13 +25,19 @@ cache
### IDE
.vscode
.idea
*.nogit.*
*.pdf
*.yaml
src/data
neo4j*
*/package-lock.json
web/package-lock.json
saves
notebooks
*.yaml
local_neo4j/data
local_neo4j/logs
local_neo4j/import
local_neo4j/plugins
local_neo4j/conf

View File

@ -22,7 +22,8 @@ zhipuai
### 【可选】配置图数据库 neo4j
使用 docker 部署 neo4j 服务,配置文件见 [local_neo4j/docker-compose.yml](local_neo4j/docker-compose.yml). 默认账号密码见最后一行,可以使用 `http://localhost:7474/` 在浏览器可视化访问。
使用 docker 部署 neo4j 服务,配置文件见 [local_neo4j/docker-compose.yml](local_neo4j/docker-compose.yml).
默认账号密码见最后一行,可以使用 `http://localhost:7474/` 在浏览器可视化访问。
```bash
cd local_neo4j
@ -30,12 +31,14 @@ docker compose up -d
```
可以使用 `python test_neo4j.py` 来测试是否正常启动。使用 `docker compose down` 可停止服务。
如果想要管理 neo4j也可以使用 `docker ps` 查看容器 id然后使用 `docker exec -it <CONTAINER_ID> /bin/bash` 进入容器。
如果想要删除数据库中的文件,可以进入容器并停止 neo4j 后,执行 `rm -rf /data/databases`
## 启动
```bash
python -m src.api
python -m src.api
cd web
npm install

View File

@ -0,0 +1,17 @@
version: '3.9'
services:
neo4j:
image: neo4j:latest
volumes:
- ./conf:/var/lib/neo4j/conf
- ./import:/var/lib/neo4j/import
- ./plugins:/plugins
- ./data:/data
- ./logs:/var/lib/neo4j/logs
restart: always
ports:
- 7474:7474
- 7687:7687
environment:
- NEO4J_AUTH=neo4j/0123456789

34
local_neo4j/test_neo4j.py Normal file
View File

@ -0,0 +1,34 @@
from neo4j import GraphDatabase
from neo4j.exceptions import ServiceUnavailable, AuthError
def check_neo4j_status(uri="bolt://localhost:7687", username="neo4j", password="0123456789"):
"""
检查 Neo4j 数据库是否可以连接并正常工作
参数:
uri (str): Neo4j URI默认为 "bolt://localhost:7687"
username (str): 数据库用户名默认为 "neo4j"
password (str): 数据库密码默认为 "0123456789"
返回:
str: "OK" 表示连接成功"UNAVAILABLE" 表示服务不可用"AUTH_FAILED" 表示认证失败
"""
try:
driver = GraphDatabase.driver(uri, auth=(username, password))
with driver.session() as session:
# 简单的查询来测试连接
result = session.run("RETURN 1")
if result.single()[0] == 1:
return "OK"
except ServiceUnavailable:
return "UNAVAILABLE"
except AuthError:
return "AUTH_FAILED"
finally:
# 确保关闭驱动
driver.close()
# 测试函数
status = check_neo4j_status()
print(f"Neo4j status: {status}")

View File

@ -71,13 +71,16 @@ class DataBaseManager:
return {"databases": [db.to_dict() for db in self.data["databases"]]}
def get_graph(self):
if self.config.enable_graph_base:
if self.config.enable_knowledge_graph:
self.data["graph"].update(self.graph_base.get_database_info("neo4j"))
return {"graph": self.data["graph"]}
else:
return {"message": "Graph base not enabled", "graph": {}}
def create_database(self, database_name, description, db_type, dimension):
from src.config import EMBED_MODEL_INFO
dimension = dimension or EMBED_MODEL_INFO[self.config.embed_model]["dimension"]
new_database = DataBaseLite(database_name,
description,
db_type,
@ -94,7 +97,7 @@ class DataBaseManager:
if db.embed_model != self.config.embed_model:
logger.error(f"Embed model not match, {db.embed_model} != {self.config.embed_model}")
return {"message": "Embed model not match", "status": "failed"}
return {"message": f"Embed model not match, cur: {self.config.embed_model}", "status": "failed"}
new_files = []
for file in files:
@ -168,7 +171,6 @@ class DataBaseManager:
logger.error(f"File format not supported, only support {support_format}")
raise Exception(f"File format not supported, only support {support_format}")
def delete_file(self, db_id, file_id):
db = self.get_kb_by_id(db_id)
file_idx_to_delete = [idx for idx, f in enumerate(db.files) if f["file_id"] == file_id][0]

View File

@ -9,11 +9,12 @@ import warnings
from src.plugins import pdf2txt
from src.plugins.oneke import OneKE
from src.utils import setup_logger
logger = setup_logger("server-graphbase")
warnings.filterwarnings("ignore", category=UserWarning)
UIE_MODEL = None
class GraphDatabase:
@ -36,6 +37,16 @@ class GraphDatabase:
"""关闭数据库连接"""
self.driver.close()
def get_sample_nodes(self, kgdb_name='neo4j', num=50):
"""获取指定数据库的前 num 个节点信息"""
self.use_database(kgdb_name)
def query(tx, num):
result = tx.run("MATCH (n)-[r]->(m) RETURN n, r, m LIMIT $num", num=int(num))
return result.values()
with self.driver.session() as session:
return session.execute_read(query, num)
def create_graph_database(self, kgdb_name):
"""创建新的数据库,如果已存在则返回已有数据库的名称"""
with self.driver.session() as session:
@ -116,21 +127,25 @@ class GraphDatabase:
MERGE (t:Entity {name: $t})
MERGE (h)-[r:RELATION {type: $r}]->(t)
""", h=entry['h'], t=entry['t'], r=entry['r'])
def _create_vector_index(tx):
index_name = "entity-embeddings"
def _create_vector_index(tx, dim):
index_name = "entityEmbeddings"
if not _index_exists(tx, index_name):
tx.run(f"""
CREATE VECTOR INDEX {index_name}
FOR (n: Entity) ON (n.embedding)
OPTIONS {{indexConfig: {{
`vector.dimensions`: 1024,
`vector.dimensions`: {dim},
`vector.similarity_function`: 'cosine'
}} }};
""")
from src.config import EMBED_MODEL_INFO
embed_info = EMBED_MODEL_INFO[self.config.embed_model]
with self.driver.session() as session:
session.execute_write(_create_graph, triples)
session.execute_write(_create_vector_index)
for entry in triples:
session.execute_write(_create_vector_index, embed_info.dimension)
for i, entry in enumerate(triples):
logger.info(f"Adding entity {i+1}/{len(triples)}")
embedding_h = self.get_embedding(entry['h'])
session.execute_write(self.set_embedding, entry['h'], embedding_h)
@ -148,37 +163,39 @@ class GraphDatabase:
triples = list(read_triples(file_path))
def batch_create(tx, triples):
query = """
UNWIND $triples AS triple
MERGE (a:Entity {name: triple.h})
MERGE (b:Entity {name: triple.t})
MERGE (a)-[r:RELATION {type: triple.r}]->(b)
"""
tx.run(query, triples=triples)
self.txt_add_vector_entity(triples, kgdb_name)
def batch_add_embeddings(tx, embeddings):
query = """
UNWIND $embeddings AS embedding
MATCH (e:Entity {name: embedding.name})
SET e.embedding = embedding.vector
"""
tx.run(query, embeddings=embeddings)
with self.driver.session() as session:
session.execute_write(batch_create, triples)
# 获取embedding并批量添加
embeddings = []
for triple in triples:
h = triple['h']
t = triple['t']
embedding_h = self.get_embedding(h)
embedding_t = self.get_embedding(t)
embeddings.append({"name": h, "vector": embedding_h})
embeddings.append({"name": t, "vector": embedding_t})
session.execute_write(batch_add_embeddings, embeddings)
# def batch_create(tx, triples):
# query = """
# UNWIND $triples AS triple
# MERGE (a:Entity {name: triple.h})
# MERGE (b:Entity {name: triple.t})
# MERGE (a)-[r:RELATION {type: triple.r}]->(b)
# """
# tx.run(query, triples=triples)
#
# def batch_add_embeddings(tx, embeddings):
# query = """
# UNWIND $embeddings AS embedding
# MATCH (e:Entity {name: embedding.name})
# SET e.embedding = embedding.vector
# """
# tx.run(query, embeddings=embeddings)
#
# with self.driver.session() as session:
# session.execute_write(batch_create, triples)
#
# # 获取embedding并批量添加
# embeddings = []
# for triple in triples:
# h = triple['h']
# t = triple['t']
# embedding_h = self.get_embedding(h)
# embedding_t = self.get_embedding(t)
# embeddings.append({"name": h, "vector": embedding_h})
# embeddings.append({"name": t, "vector": embedding_t})
#
# session.execute_write(batch_add_embeddings, embeddings)
self.status = "open"
return kgdb_name
@ -260,13 +277,21 @@ class GraphDatabase:
with self.driver.session() as session:
return session.execute_read(query, keyword, hops)
def query_node(self, entity_name, args):
# TODO 添加判断节点数量为 0 停止检索
if args.get("exact_match"):
raise NotImplemented("not implement for `exact_match`")
else:
return self.query_by_vector(entity_name, kgdb_name=args.get("kgdb_name"), hops=args.get("hops"))
def query_by_vector_tep(self, keyword, kgdb_name='neo4j'):
"""向量查询"""
self.use_database(kgdb_name)
def query(tx, text):
embedding = self.get_embedding(text)
result = tx.run("""
CALL db.index.vector.queryNodes('entity-embeddings', 10, $embedding)
CALL db.index.vector.queryNodes('entityEmbeddings', 10, $embedding)
YIELD node AS similarEntity, score
RETURN similarEntity.name AS name, score
""", embedding=embedding)
@ -277,7 +302,7 @@ class GraphDatabase:
with self.driver.session() as session:
return session.execute_read(query, keyword)
def query_by_vector(self, entity_name, threshold=0.9,kgdb_name='neo4j', hops=2, num_of_res=2):
def query_by_vector(self, entity_name, threshold=0.9, kgdb_name='neo4j', hops=2, num_of_res=2):
self.use_database(kgdb_name)
result = self.query_by_vector_tep(entity_name)
querys = []

View File

@ -83,7 +83,7 @@ class Retriever:
r["file"] = kb.id2file(r["entity"]["file_id"])
if self.config.enable_reranker:
RERANK_THRESHOLD = 0.1
RERANK_THRESHOLD = 0.001
for r in kb_res:
r["rerank_score"] = self.reranker.compute_score([query, r["entity"]["text"]], normalize=True)
kb_res.sort(key=lambda x: x["rerank_score"], reverse=True)
@ -124,7 +124,46 @@ class Retriever:
return entities
def foramt_general_results(self, results):
logger.debug(f"Formatting general results: {results}")
formatted_results = {"nodes": [], "edges": []}
for item in results:
relationship = item[1]
rel_id = relationship.element_id
nodes = relationship.nodes
if len(nodes) != 2:
continue
source, target = nodes
source_id = source.element_id
target_id = target.element_id
source_name = source._properties.get('name', 'unknown')
target_name = target._properties.get('name', 'unknown')
if source_id not in formatted_results["nodes"]:
formatted_results["nodes"].append({"id": source_id, "name": source_name})
if target_id not in formatted_results["nodes"]:
formatted_results["nodes"].append({"id": target_id, "name": target_name})
relationship_type = relationship._properties.get('type', 'unknown')
if relationship_type == 'unknown':
relationship_type = relationship.type
formatted_results["edges"].append({
"id": rel_id,
"type": relationship_type,
"source_id": source_id,
"target_id": target_id,
"source_name": source_name,
"target_name": target_name
})
return formatted_results
def format_query_results(self, results):
logger.debug(f"Formatting query results: {results}")
formatted_results = {"nodes": [], "edges": []}
node_dict = {}

View File

@ -45,10 +45,11 @@ class ZhipuEmbedding:
self.query_instruction_for_retrieval = "为这个句子生成表示以用于检索相关文章:"
def predict(self, message):
data = []
for i in range(0, len(message), 10):
if len(message) > 10:
logger.info(f"Encoding {i} to {i+10} with {len(message)} messages")
group_msg = message[i:i+10]
response = self.client.embeddings.create(
model=self.model_info.default_path,

View File

@ -123,9 +123,20 @@ def get_graph_node():
return jsonify({'message': 'entity_name and kgdb_name are required'}), 400
logger.debug(f"Get graph node {entity_name} in {kgdb_name} with {hops} hops")
result = startup.dbm.graph_base.query_by_vector(entity_name, kgdb_name=kgdb_name, hops=hops)
result = startup.dbm.graph_base.query_node(entity_name, request.args)
return jsonify({'result': startup.retriever.format_query_results(result), 'message': 'success'}), 200
@db.route('/graph/nodes', methods=['GET'])
def get_graph_nodes():
kgdb_name = request.args.get('kgdb_name')
num = request.args.get('num')
if not kgdb_name:
return jsonify({'message': 'kgdb_name is required'}), 400
logger.debug(f"Get graph nodes in {kgdb_name} with {num} nodes")
result = startup.dbm.graph_base.get_sample_nodes(kgdb_name, num)
return jsonify({'result': startup.retriever.foramt_general_results(result), 'message': 'success'}), 200
@db.route('/graph/add', methods=['POST'])
def add_graph_entity():
data = json.loads(request.data)

View File

@ -8,7 +8,7 @@
.layout-container {
width: 100%;
padding: 16px 30px;
padding: 0px 30px;
background-color: #FCFEFF;
h2 {

View File

@ -107,11 +107,12 @@
:class="message.role"
>
<p v-if="message.role=='sent'" style="white-space: pre-line" class="message-text">{{ message.text }}</p>
<div v-else-if="message.text.length == 0" class="loading-dots">
<div v-else-if="message.text.length == 0 && message.status=='querying'" class="loading-dots">
<div></div>
<div></div>
<div></div>
</div>
<div v-else-if="message.text.length == 0 || message.status == 'error'" class="err-msg">请求错误请重试</div>
<p v-else
v-html="renderMarkdown(message)"
class="message-md"
@ -182,7 +183,8 @@ const examples = ref([
'肉碱的分子量是多少?直接回答',
'简述大蒜的功效是什么?',
'A大于BB小于CA和C哪个大',
'今天天气怎么样?'
'今天天气怎么样?',
'吃饭吃出苍蝇可以索赔吗?',
])
const opts = reactive({
@ -404,6 +406,11 @@ const sendMessage = () => {
}
return readChunk()
})
.catch((error) => {
console.error(error)
updateStatus(cur_res_id, "error")
isStreaming.value = false
})
} else {
console.log('请输入消息')
}
@ -592,6 +599,15 @@ watch(
color: black;
/* box-shadow: 0px 0.3px 0.9px rgba(0, 0, 0, 0.12), 0px 1.6px 3.6px rgba(0, 0, 0, 0.16); */
/* animation: slideInUp 0.1s ease-in; */
.err-msg {
color: red;
border: 1px solid red;
padding: 0.2rem 1rem;
border-radius: 8px;
text-align: center;
background: #FFEBEE;
}
}
.message-box.sent {
@ -624,8 +640,6 @@ watch(
word-wrap: break-word;
margin-bottom: 0;
}
}

View File

@ -1,8 +1,8 @@
<!-- RefsComponent.vue -->
<template>
<div class="refs" v-if="showRefs">
<span class="item"><GlobalOutlined /> {{ message.model_name }}</span>
<div class="tags">
<span class="item"><GlobalOutlined /> {{ message.model_name }}</span>
<span class="filetag item"
v-for="(results, filename) in message.groupedResults"
:key="filename"
@ -81,6 +81,7 @@ const showRefs = computed(() => message.value.role=='received' && message.value.
.tags {
display: flex;
flex-wrap: wrap;
gap: 10px;
.filetag {

View File

@ -1,9 +1,24 @@
<template>
<div class="graph-container layout-container" v-if="state.showPage">
<div class="database-empty" v-if="!state.showPage">
<a-empty>
<template #description>
<span>
前往 <router-link to="/setting" style="color: var(--main-color); font-weight: bold;">设置</router-link>
</span>
</template>
</a-empty>
</div>
<div class="graph-container layout-container" v-else>
<div class="info">
<h2>Neo4j 图数据库</h2>
<p>基于 Neo4j 构建的图数据库</p>
</div>
<h2>图数据库 {{ graph?.database_name }}</h2>
<p>
<span v-if="state.graphloading">加载中</span>
<span class="green-dot" v-if="graph?.status == 'open'"></span>
<span class="red-dot" v-else></span>
<span>{{ graph?.status }}</span> ·
<span> {{ graph?.entity_count }} 实体{{ graph?.relationship_count }} 个关系</span>
</p>
</div>
<div class="actions">
<div class="actions-left">
<a-button @click="state.showModal = true">上传文件</a-button>
@ -32,6 +47,8 @@
</a-upload-dragger>
</div>
</a-modal>
<input v-model="sampleNodeCount">
<a-button @click="loadSampleNodes">确定</a-button>
</div>
<div class="action-right">
<input
@ -49,16 +66,7 @@
</a-button>
</div>
</div>
<div class="main" id="container"></div>
</div>
<div class="database-empty" v-else>
<a-empty>
<template #description>
<span>
前往 <router-link to="/setting" style="color: var(--main-color); font-weight: bold;">设置</router-link>
</span>
</template>
</a-empty>
<div class="main" id="container" ref="container"></div>
</div>
</template>
@ -71,24 +79,17 @@ import { useConfigStore } from '@/stores/config';
const configStore = useConfigStore()
let graphInstance
const graph = ref(null)
const container = ref(null);
const fileList = ref([]);
const sampleNodeCount = ref(100);
const subgraph = reactive({
nodes: [
{ id: '1', name: 'node1' },
{ id: '2', name: 'node2' },
{ id: '3', name: 'node3' },
{ id: '4', name: 'node4' },
{ id: '5', name: 'node5' },
],
edges: [
{ id: 'e1', source_id: '1', target_id: '2', type: 'edge1' },
{ id: 'e2', source_id: '1', target_id: '3', type: 'edge2' },
{ id: 'e3', source_id: '2', target_id: '4', type: 'edge3' },
{ id: 'e4', source_id: '2', target_id: '5', type: 'edge4' },
],
nodes: [],
edges: [],
});
const state = reactive({
graphloading: false,
searchInput: '',
searchLoading: false,
showModal: false,
@ -96,10 +97,24 @@ const state = reactive({
showPage: computed(() => configStore.config.enable_knowledge_base && configStore.config.enable_knowledge_graph),
})
// const showPage = computed(() => {
// return configStore.config.enable_knowledge_base && configStore.config.enable_knowledge_graph
// })
const loadGraph = () => {
state.graphloading = true
fetch('/api/database/graph', {
method: "GET",
})
.then(response => response.json())
.then(data => {
console.log(data)
graph.value = data.graph
state.graphloading = false
})
.catch(error => {
console.error(error)
message.error(error.message)
state.graphloading = false
})
}
const graphData = computed(() => {
return {
@ -143,6 +158,29 @@ const addDocumentByFile = () => {
.finally(() => state.precessing = false)
};
const loadSampleNodes = () => {
fetch(`/api/database/graph/nodes?kgdb_name=neo4j&num=${sampleNodeCount.value}`)
.then((res) => {
if (res.ok) {
return res.json();
} else {
throw new Error("加载失败");
}
})
.then((data) => {
subgraph.nodes = data.result.nodes
subgraph.edges = data.result.edges
console.log(data)
console.log(subgraph)
setTimeout(() => {
randerGraph()
}, 500)
})
.catch((error) => {
message.error(error.message);
})
}
const onSearch = () => {
if (!state.searchInput) {
message.error('请输入要查询的实体')
@ -177,44 +215,49 @@ const randerGraph = () => {
}
onMounted(() => {
if (!state.showPage) {
return
}
graphInstance = new Graph({
container: document.getElementById("container"),
width: document.getElementById("container").offsetWidth,
height: document.getElementById("container").offsetHeight,
autoFit: true,
autoResize: true,
layout: {
type: 'force-atlas2',
preventOverlap: true,
kr: 100,
},
node: {
type: 'circle',
style: {
labelText: (d) => d.data.label,
size: 40,
},
palette: {
field: 'label',
color: 'tableau',
},
},
edge: {
type: 'line',
style: {
labelText: (d) => d.data.label,
labelBackground: '#fff',
},
},
behaviors: ['drag-element'],
});
graphInstance.setData(graphData.value);
graphInstance.render();
window.addEventListener('resize', randerGraph);
loadGraph();
loadSampleNodes();
setTimeout(() => {
if (state.showPage) {
graphInstance = new Graph({
container: container.value,
width: container.value.offsetWidth,
height: container.value.offsetHeight,
autoFit: true,
autoResize: true,
layout: {
type: 'd3-force',
preventOverlap: true,
kr: 100,
collide: {
strength: 0.5,
},
},
node: {
type: 'circle',
style: {
labelText: (d) => d.data.label,
size: 40,
},
palette: {
field: 'label',
color: 'tableau',
},
},
edge: {
type: 'line',
style: {
labelText: (d) => d.data.label,
labelBackground: '#fff',
},
},
behaviors: ['drag-element', 'zoom-canvas', 'drag-canvas'],
});
graphInstance.setData(graphData.value);
graphInstance.render();
window.addEventListener('resize', randerGraph);
}
}, 400)
});
@ -231,16 +274,47 @@ const handleDrop = (event) => {
</script>
<style lang="less" scoped>
.graph-container {
.info span.green-dot, .info span.red-dot {
display: inline-block;
width: 14px;
height: 14px;
border-radius: 50%;
margin: 0 5px;
}
.info span.green-dot {
background: #52c41a;
}
.info span.red-dot {
background: #f5222d;
}
.info {
margin-bottom: 20px;
}
}
.actions {
display: flex;
justify-content: space-between;
margin-bottom: 20px;
.actions-left {
display: flex;
align-items: center;
gap: 10px;
}
input {
width: 100px;
margin-right: 10px;
border-radius: 8px;
padding: 4px 12px;
border: 2px solid #d9d9d9;
border: 2px solid var(--main-300);
outline: none;
height: 42px;
@ -269,7 +343,8 @@ const handleDrop = (event) => {
margin: 20px 0;
border-radius: 16px;
width: 100%;
height: calc(100% - 200px);
height: 800px;
resize: horizontal;
}
.database-empty {

View File

@ -137,9 +137,15 @@ const state = reactive({
const handleChange = (key, e) => {
if (key == 'enable_knowledge_graph' && e && !configStore.config.enable_knowledge_base) {
message.error('请先启用知识库功能')
message.error('启动知识图谱必须请先启用知识库功能')
return
}
if (key == 'enable_knowledge_base' && !e && configStore.config.enable_knowledge_graph) {
message.error('关闭知识库功能必须请先关闭知识图谱功能')
return
}
console.log('Change', key, e)
needRestart[key] = true
configStore.setConfigValue(key, e)