fix(graph): 改进图数据库查询逻辑和错误处理
修复图数据库查询时的错误处理,添加完整的traceback信息 重构查询逻辑,分离向量查询和模糊查询方法 更新README文档说明知识图谱的限制和使用方法 确保查询只针对带有Entity标签的节点 添加数据库状态检查断言
This commit is contained in:
parent
b4439996fa
commit
007e6aeb1f
@ -153,19 +153,23 @@ custom-provider-name-here:
|
|||||||
|
|
||||||
### 知识图谱
|
### 知识图谱
|
||||||
|
|
||||||
在 v0.2 版本中,项目支持了基于 [LightRAG](https://github.com/HKUDS/LightRAG) 的知识图谱构建方法。需要在知识库中创建一个基于 LightRAG 的知识库,然后上传文档。构建的知识图谱会自动导入到 Neo4j 中,并使用不同的 label 做区分。
|
在 v0.2 版本中,项目支持了基于 [LightRAG](https://github.com/HKUDS/LightRAG) 的知识图谱构建方法。需要在知识库中创建一个基于 LightRAG 的知识库,然后上传文档。构建的知识图谱会自动导入到 Neo4j 中,并使用不同的 label 做区分。需要说明的是,基于 LightRAG 的知识库,可以在知识库详情中可视化,但是不能在侧边栏的图谱中检索,知识图谱检索工具也不支持基于 LightRAG 的知识库进行检索。基于 LightRAG 方法构建的图谱的查询,需要使用对应的知识库作为查询工具。
|
||||||
|
|
||||||
|知识库可视化|Neo4J管理端|
|
|知识库可视化|Neo4J管理端|
|
||||||
|--|--|
|
|--|--|
|
||||||
|||
|
|||
|
||||||
|
|
||||||
除此之外,也支持将已有的知识图谱按照下面的格式导入 Neo4j 中,或者通过修改 `docker-compose.yml` 中的 `NEO4J_URI` 配置来接入已有的 Neo4j 实例。默认账户密码是`neo4j` / `0123456789`。
|
除此之外,也支持将已有的知识图谱按照下面的格式导入 Neo4j 中,上传后,节点会自动添加 `Upload`、`Entity` 标签,关系会自动添加 `Relation` 标签。可以通过 `name` 属性访问实体的名称,使用 `type` 属性访问边的名称。默认账户密码是`neo4j` / `0123456789`。
|
||||||
|
|
||||||
**数据格式**:支持 JSONL 格式导入
|
**数据格式**:支持 JSONL 格式导入
|
||||||
```jsonl
|
```jsonl
|
||||||
{"h": "北京", "t": "中国", "r": "首都"}
|
{"h": "北京", "t": "中国", "r": "首都"}
|
||||||
{"h": "上海", "t": "中国", "r": "直辖市"}
|
{"h": "上海", "t": "中国", "r": "直辖市"}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
此外,也可以通过修改 `docker-compose.yml` 中的 `NEO4J_URI` 配置来接入已有的 Neo4j 实例,但是最好确保每个节点都有 Entity 标签,否则会影响到图的检索与构建。
|
||||||
|
|
||||||
|
|
||||||
## 🔧 高级配置
|
## 🔧 高级配置
|
||||||
|
|
||||||
### OCR 服务(可选)
|
### OCR 服务(可选)
|
||||||
|
|||||||
@ -223,7 +223,7 @@ async def get_neo4j_node(
|
|||||||
}
|
}
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"查询图节点失败: {e}")
|
logger.error(f"查询图节点失败: {e}\n{traceback.format_exc()}")
|
||||||
raise HTTPException(status_code=500, detail=f"查询图节点失败: {str(e)}")
|
raise HTTPException(status_code=500, detail=f"查询图节点失败: {str(e)}")
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|||||||
@ -53,14 +53,15 @@ class GraphDatabase:
|
|||||||
|
|
||||||
def is_running(self):
|
def is_running(self):
|
||||||
"""检查图数据库是否正在运行"""
|
"""检查图数据库是否正在运行"""
|
||||||
return self.status == "open"
|
return self.status == "open" or self.status == "processing"
|
||||||
|
|
||||||
def get_sample_nodes(self, kgdb_name='neo4j', num=50):
|
def get_sample_nodes(self, kgdb_name='neo4j', num=50):
|
||||||
"""获取指定数据库的 num 个节点信息"""
|
"""获取指定数据库的 num 个节点信息"""
|
||||||
assert self.driver is not None, "Database is not connected"
|
assert self.driver is not None, "Database is not connected"
|
||||||
self.use_database(kgdb_name)
|
self.use_database(kgdb_name)
|
||||||
def query(tx, num):
|
def query(tx, num):
|
||||||
result = tx.run("MATCH (n)-[r]->(m) RETURN n, r, m LIMIT $num", num=int(num))
|
"""Note: 这里只查询带有 Entity 标签的节点"""
|
||||||
|
result = tx.run("MATCH (n:Entity)-[r]->(m:Entity) RETURN n, r, m LIMIT $num", num=int(num))
|
||||||
return result.values()
|
return result.values()
|
||||||
|
|
||||||
with self.driver.session() as session:
|
with self.driver.session() as session:
|
||||||
@ -97,8 +98,8 @@ class GraphDatabase:
|
|||||||
t = triple['t']
|
t = triple['t']
|
||||||
r = triple['r']
|
r = triple['r']
|
||||||
query = (
|
query = (
|
||||||
"MERGE (a:Entity {name: $h}) "
|
"MERGE (a:Entity:Upload {name: $h}) "
|
||||||
"MERGE (b:Entity {name: $t}) "
|
"MERGE (b:Entity:Upload {name: $t}) "
|
||||||
"MERGE (a)-[:" + r.replace(" ", "_") + "]->(b)"
|
"MERGE (a)-[:" + r.replace(" ", "_") + "]->(b)"
|
||||||
)
|
)
|
||||||
tx.run(query, h=h, t=t)
|
tx.run(query, h=h, t=t)
|
||||||
@ -122,8 +123,8 @@ class GraphDatabase:
|
|||||||
"""添加一个三元组"""
|
"""添加一个三元组"""
|
||||||
for entry in data:
|
for entry in data:
|
||||||
tx.run("""
|
tx.run("""
|
||||||
MERGE (h:Entity {name: $h})
|
MERGE (h:Entity:Upload {name: $h})
|
||||||
MERGE (t:Entity {name: $t})
|
MERGE (t:Entity:Upload {name: $t})
|
||||||
MERGE (h)-[r:RELATION {type: $r}]->(t)
|
MERGE (h)-[r:RELATION {type: $r}]->(t)
|
||||||
""", h=entry['h'], t=entry['t'], r=entry['r'])
|
""", h=entry['h'], t=entry['t'], r=entry['r'])
|
||||||
|
|
||||||
@ -167,10 +168,11 @@ class GraphDatabase:
|
|||||||
""", name=entity_name, embedding=embedding)
|
""", name=entity_name, embedding=embedding)
|
||||||
|
|
||||||
# 判断模型名称是否匹配
|
# 判断模型名称是否匹配
|
||||||
cur_embed_info = config.embed_model_names[config.embed_model]
|
self.embed_model_name = self.embed_model_name or config.embed_model
|
||||||
self.embed_model_name = self.embed_model_name or cur_embed_info.get('name')
|
cur_embed_info = config.embed_model_names.get(self.embed_model_name)
|
||||||
assert self.embed_model_name == cur_embed_info.get('name') or self.embed_model_name is None, \
|
logger.warning(f"embed_model_name={self.embed_model_name}, {cur_embed_info=}")
|
||||||
f"embed_model_name={self.embed_model_name}, {cur_embed_info.get('name')=}"
|
assert self.embed_model_name == config.embed_model or self.embed_model_name is None, \
|
||||||
|
f"embed_model_name={self.embed_model_name}, {config.embed_model=}"
|
||||||
|
|
||||||
with self.driver.session() as session:
|
with self.driver.session() as session:
|
||||||
logger.info(f"Adding entity to {kgdb_name}")
|
logger.info(f"Adding entity to {kgdb_name}")
|
||||||
@ -267,12 +269,48 @@ class GraphDatabase:
|
|||||||
def query_node(self, entity_name, threshold=0.9, kgdb_name='neo4j', hops=2, max_entities=5, **kwargs):
|
def query_node(self, entity_name, threshold=0.9, kgdb_name='neo4j', hops=2, max_entities=5, **kwargs):
|
||||||
"""知识图谱查询节点的入口:"""
|
"""知识图谱查询节点的入口:"""
|
||||||
assert self.driver is not None, "Database is not connected"
|
assert self.driver is not None, "Database is not connected"
|
||||||
# TODO 添加判断节点数量为 0 停止检索
|
assert self.is_running(), "图数据库未启动"
|
||||||
# 判断是否启动
|
|
||||||
if not self.is_running():
|
|
||||||
raise Exception("图数据库未启动")
|
|
||||||
|
|
||||||
self.use_database(kgdb_name)
|
self.use_database(kgdb_name)
|
||||||
|
|
||||||
|
# 使用向量索引进行查询
|
||||||
|
results_sim = self._query_with_vector_sim(entity_name, kgdb_name, hops, threshold)
|
||||||
|
results_fuzzy = self._query_with_fuzzy_match(entity_name, kgdb_name, hops)
|
||||||
|
results = results_sim + results_fuzzy
|
||||||
|
qualified_entities = [result[0] for result in results][:max_entities]
|
||||||
|
|
||||||
|
logger.debug(f"Graph Query Entities: {entity_name}, {qualified_entities=}")
|
||||||
|
|
||||||
|
# 对每个合格的实体进行查询
|
||||||
|
all_query_results = []
|
||||||
|
for entity in qualified_entities:
|
||||||
|
query_result = self._query_specific_entity(entity_name=entity, hops=hops, kgdb_name=kgdb_name)
|
||||||
|
all_query_results.extend(query_result)
|
||||||
|
|
||||||
|
return all_query_results
|
||||||
|
|
||||||
|
def _query_with_fuzzy_match(self, keyword, kgdb_name='neo4j', hops = 2):
|
||||||
|
"""模糊查询"""
|
||||||
|
assert self.driver is not None, "Database is not connected"
|
||||||
|
self.use_database(kgdb_name)
|
||||||
|
def query_fuzzy_match(tx, keyword, hops):
|
||||||
|
result = tx.run("""
|
||||||
|
MATCH (n:Entity)
|
||||||
|
WHERE n.name CONTAINS $keyword
|
||||||
|
RETURN DISTINCT n.name AS name
|
||||||
|
""", keyword=keyword)
|
||||||
|
values = result.values()
|
||||||
|
logger.debug(f"Fuzzy Query Results: {values}")
|
||||||
|
return values
|
||||||
|
|
||||||
|
with self.driver.session() as session:
|
||||||
|
return session.execute_read(query_fuzzy_match, keyword, hops)
|
||||||
|
|
||||||
|
def _query_with_vector_sim(self, keyword, kgdb_name='neo4j', hops = 2, threshold=0.9):
|
||||||
|
"""向量查询"""
|
||||||
|
assert self.driver is not None, "Database is not connected"
|
||||||
|
self.use_database(kgdb_name)
|
||||||
|
|
||||||
def _index_exists(tx, index_name):
|
def _index_exists(tx, index_name):
|
||||||
"""检查索引是否存在"""
|
"""检查索引是否存在"""
|
||||||
result = tx.run("SHOW INDEXES")
|
result = tx.run("SHOW INDEXES")
|
||||||
@ -281,7 +319,7 @@ class GraphDatabase:
|
|||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def query(tx, text):
|
def query_by_vector(tx, text, threshold):
|
||||||
# 首先检查索引是否存在
|
# 首先检查索引是否存在
|
||||||
if not _index_exists(tx, "entityEmbeddings"):
|
if not _index_exists(tx, "entityEmbeddings"):
|
||||||
raise Exception("向量索引不存在,请先创建索引")
|
raise Exception("向量索引不存在,请先创建索引")
|
||||||
@ -292,30 +330,15 @@ class GraphDatabase:
|
|||||||
YIELD node AS similarEntity, score
|
YIELD node AS similarEntity, score
|
||||||
RETURN similarEntity.name AS name, score
|
RETURN similarEntity.name AS name, score
|
||||||
""", embedding=embedding)
|
""", embedding=embedding)
|
||||||
return result.values()
|
return [r for r in result if r["score"] > threshold]
|
||||||
|
|
||||||
try:
|
with self.driver.session() as session:
|
||||||
with self.driver.session() as session:
|
results = session.execute_read(query_by_vector, keyword, threshold=threshold)
|
||||||
results = session.execute_read(query, entity_name)
|
results = clean_triples_embedding(results)
|
||||||
except Exception as e:
|
return results
|
||||||
if "向量索引不存在" in str(e):
|
|
||||||
logger.error(f"向量索引不存在,请先创建索引: {e}, {traceback.format_exc()}")
|
|
||||||
return []
|
|
||||||
raise e
|
|
||||||
|
|
||||||
# 筛选出分数高于阈值的实体
|
|
||||||
qualified_entities = [result[0] for result in results[:max_entities] if result[1] > threshold]
|
|
||||||
logger.debug(f"Graph Query Entities: {entity_name}, {qualified_entities=}")
|
|
||||||
|
|
||||||
# 对每个合格的实体进行查询
|
def _query_specific_entity(self, entity_name, kgdb_name='neo4j', hops=2, limit=100):
|
||||||
all_query_results = []
|
|
||||||
for entity in qualified_entities:
|
|
||||||
query_result = self.query_specific_entity(entity_name=entity, hops=hops, kgdb_name=kgdb_name)
|
|
||||||
all_query_results.extend(query_result)
|
|
||||||
|
|
||||||
return all_query_results
|
|
||||||
|
|
||||||
def query_specific_entity(self, entity_name, kgdb_name='neo4j', hops=2, limit=100):
|
|
||||||
"""查询指定实体三元组信息(无向关系)"""
|
"""查询指定实体三元组信息(无向关系)"""
|
||||||
assert self.driver is not None, "Database is not connected"
|
assert self.driver is not None, "Database is not connected"
|
||||||
if not entity_name:
|
if not entity_name:
|
||||||
@ -355,6 +378,7 @@ class GraphDatabase:
|
|||||||
|
|
||||||
def query_all_nodes_and_relationships(self, kgdb_name='neo4j', hops = 2):
|
def query_all_nodes_and_relationships(self, kgdb_name='neo4j', hops = 2):
|
||||||
"""查询图数据库中所有三元组信息 NEVER USE"""
|
"""查询图数据库中所有三元组信息 NEVER USE"""
|
||||||
|
raise Exception("NEVER USE")
|
||||||
assert self.driver is not None, "Database is not connected"
|
assert self.driver is not None, "Database is not connected"
|
||||||
self.use_database(kgdb_name)
|
self.use_database(kgdb_name)
|
||||||
def query(tx, hops):
|
def query(tx, hops):
|
||||||
@ -371,6 +395,7 @@ class GraphDatabase:
|
|||||||
|
|
||||||
def query_by_relationship_type(self, relationship_type, kgdb_name='neo4j', hops = 2):
|
def query_by_relationship_type(self, relationship_type, kgdb_name='neo4j', hops = 2):
|
||||||
"""查询指定关系三元组信息 NEVER USE"""
|
"""查询指定关系三元组信息 NEVER USE"""
|
||||||
|
raise Exception("NEVER USE")
|
||||||
assert self.driver is not None, "Database is not connected"
|
assert self.driver is not None, "Database is not connected"
|
||||||
self.use_database(kgdb_name)
|
self.use_database(kgdb_name)
|
||||||
def query(tx, relationship_type, hops):
|
def query(tx, relationship_type, hops):
|
||||||
@ -385,26 +410,9 @@ class GraphDatabase:
|
|||||||
with self.driver.session() as session:
|
with self.driver.session() as session:
|
||||||
return session.execute_read(query, relationship_type, hops)
|
return session.execute_read(query, relationship_type, hops)
|
||||||
|
|
||||||
def query_entity_like(self, keyword, kgdb_name='neo4j', hops = 2):
|
|
||||||
"""模糊查询 NEVER USE"""
|
|
||||||
assert self.driver is not None, "Database is not connected"
|
|
||||||
self.use_database(kgdb_name)
|
|
||||||
def query(tx, keyword, hops):
|
|
||||||
result = tx.run(f"""
|
|
||||||
MATCH (n:Entity)
|
|
||||||
WHERE n.name CONTAINS $keyword
|
|
||||||
MATCH (n)-[r*1..{hops}]->(m)
|
|
||||||
RETURN n AS n, r, m AS m
|
|
||||||
""", keyword=keyword)
|
|
||||||
values = result.values()
|
|
||||||
values = clean_triples_embedding(values)
|
|
||||||
return values
|
|
||||||
|
|
||||||
with self.driver.session() as session:
|
|
||||||
return session.execute_read(query, keyword, hops)
|
|
||||||
|
|
||||||
def query_node_info(self, node_name, kgdb_name='neo4j', hops = 2):
|
def query_node_info(self, node_name, kgdb_name='neo4j', hops = 2):
|
||||||
"""查询指定节点的详细信息返回信息 NEVER USE"""
|
"""查询指定节点的详细信息返回信息 NEVER USE"""
|
||||||
|
raise Exception("NEVER USE")
|
||||||
assert self.driver is not None, "Database is not connected"
|
assert self.driver is not None, "Database is not connected"
|
||||||
self.use_database(kgdb_name) # 切换到指定数据库
|
self.use_database(kgdb_name) # 切换到指定数据库
|
||||||
def query(tx, node_name, hops):
|
def query(tx, node_name, hops):
|
||||||
@ -465,7 +473,7 @@ class GraphDatabase:
|
|||||||
}
|
}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if self.status == "open" and self.driver and self.is_running():
|
if self.is_running():
|
||||||
# 获取数据库信息
|
# 获取数据库信息
|
||||||
with self.driver.session() as session:
|
with self.driver.session() as session:
|
||||||
graph_info = session.execute_read(query)
|
graph_info = session.execute_read(query)
|
||||||
@ -474,6 +482,9 @@ class GraphDatabase:
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
graph_info["last_updated"] = datetime.now().isoformat()
|
graph_info["last_updated"] = datetime.now().isoformat()
|
||||||
return graph_info
|
return graph_info
|
||||||
|
else:
|
||||||
|
logger.warning(f"图数据库未连接或未运行:{self.status=}")
|
||||||
|
return None
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"获取图数据库信息失败:{e}, {traceback.format_exc()}")
|
logger.error(f"获取图数据库信息失败:{e}, {traceback.format_exc()}")
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user