ForcePilot/src/knowledge/graphbase.py

715 lines
29 KiB
Python
Raw Normal View History

import os
2024-07-15 18:58:34 +08:00
import json
import warnings
2025-03-19 01:21:51 +08:00
import traceback
2024-07-16 18:14:27 +08:00
from neo4j import GraphDatabase as GD
from neo4j import Query
2024-07-20 20:30:32 +08:00
from src import config
from src.models import select_embedding_model
2025-02-28 00:38:59 +08:00
from src.utils import logger
2024-09-06 12:54:17 +08:00
warnings.filterwarnings("ignore", category=UserWarning)
2024-09-09 17:07:03 +08:00
2024-07-24 18:32:14 +08:00
UIE_MODEL = None
2024-07-16 18:14:27 +08:00
class GraphDatabase:
def __init__(self):
2024-07-16 18:14:27 +08:00
self.driver = None
self.files = []
self.status = "closed"
self.kgdb_name = "neo4j"
self.embed_model_name = os.getenv("GRAPH_EMBED_MODEL_NAME") or "siliconflow/BAAI/bge-m3"
self.embed_model = select_embedding_model(self.embed_model_name)
self.work_dir = os.path.join(config.save_dir, "knowledge_graph", self.kgdb_name)
2025-02-28 00:38:59 +08:00
os.makedirs(self.work_dir, exist_ok=True)
2025-02-28 02:40:34 +08:00
# 尝试加载已保存的图数据库信息
2025-03-11 14:18:48 +08:00
if not self.load_graph_info():
logger.debug("创建新的图数据库配置")
2025-02-28 02:40:34 +08:00
2025-02-28 00:38:59 +08:00
self.start()
2024-07-16 18:14:27 +08:00
def start(self):
2024-09-11 01:08:13 +08:00
uri = os.environ.get("NEO4J_URI", "bolt://localhost:7687")
username = os.environ.get("NEO4J_USERNAME", "neo4j")
password = os.environ.get("NEO4J_PASSWORD", "0123456789")
logger.info(f"Connecting to Neo4j: {uri}/{self.kgdb_name}")
2024-10-08 22:16:17 +08:00
try:
self.driver = GD.driver(f"{uri}/{self.kgdb_name}", auth=(username, password))
self.status = "open"
logger.info(f"Connected to Neo4j: {self.get_graph_info(self.kgdb_name)}")
2025-02-28 02:40:34 +08:00
# 连接成功后保存图数据库信息
self.save_graph_info(self.kgdb_name)
2024-10-08 22:16:17 +08:00
except Exception as e:
logger.error(f"Failed to connect to Neo4j: {e}, {uri}, {self.kgdb_name}, {username}, {password}")
2024-07-15 18:58:34 +08:00
def close(self):
"""关闭数据库连接"""
assert self.driver is not None, "Database is not connected"
2024-07-15 18:58:34 +08:00
self.driver.close()
def is_running(self):
"""检查图数据库是否正在运行"""
return self.status == "open" or self.status == "processing"
2024-09-06 12:54:17 +08:00
def get_sample_nodes(self, kgdb_name='neo4j', num=50):
"""获取指定数据库的 num 个节点信息,优先返回连通的节点子图"""
assert self.driver is not None, "Database is not connected"
2024-09-06 12:54:17 +08:00
self.use_database(kgdb_name)
def query(tx, num):
"""Note: 使用连通性查询获取集中的节点子图"""
# 首先尝试获取一个连通的子图
query_str = """
// 获取高度数节点作为种子节点
MATCH (seed:Entity)
WITH seed, COUNT{(seed)-[]->()} + COUNT{(seed)<-[]-()} as degree
WHERE degree > 0
ORDER BY degree DESC
LIMIT 5
// 为每个种子节点控制连接数量避免过度集中
UNWIND seed as s
MATCH (s)-[*1..1]-(neighbor:Entity)
WITH s, neighbor, COUNT{(s)-[]->()} + COUNT{(s)<-[]-()} as s_degree
WITH s, s_degree, collect(neighbor) as neighbors
WITH s, s_degree, neighbors[0..toInteger($num * 0.1)] as limited_neighbors
// 从邻居节点扩展到二跳节点形成开枝散叶结构
UNWIND limited_neighbors as neighbor
OPTIONAL MATCH (neighbor)-[*1..1]-(second_hop:Entity)
WHERE second_hop <> s
WITH s, limited_neighbors, neighbor, collect(DISTINCT second_hop)[0..2] as second_hops
// 收集所有连通节点
WITH collect(DISTINCT s) as seeds,
collect(DISTINCT neighbor) as neighbors,
reduce(acc = [], x IN collect(second_hops) | acc + x) as second_hop_nodes
WITH seeds + neighbors + second_hop_nodes as connected_nodes
// 只使用连接的节点不添加随机节点
WITH connected_nodes[0..$num] as final_nodes
// 获取这些节点之间的关系避免双向边
UNWIND final_nodes as n
MATCH (n)-[rel]-(m)
WHERE m IN final_nodes AND elementId(n) < elementId(m)
RETURN
{id: elementId(n), name: n.name} AS h,
{type: rel.type, source_id: elementId(n), target_id: elementId(m)} AS r,
{id: elementId(m), name: m.name} AS t
"""
try:
results = tx.run(query_str, num=int(num))
formatted_results = {'nodes': [], 'edges': []}
node_ids = set()
for item in results:
h_node = item['h']
t_node = item['t']
# 避免重复添加节点
if h_node['id'] not in node_ids:
formatted_results['nodes'].append(h_node)
node_ids.add(h_node['id'])
if t_node['id'] not in node_ids:
formatted_results['nodes'].append(t_node)
node_ids.add(t_node['id'])
formatted_results['edges'].append(item['r'])
# 如果连通查询没有返回足够的结果,回退到原始查询
if len(formatted_results['nodes']) < num // 2:
fallback_query = """
MATCH (n:Entity)-[r]-(m:Entity)
WHERE elementId(n) < elementId(m)
RETURN
{id: elementId(n), name: n.name} AS h,
{type: r.type, source_id: elementId(n), target_id: elementId(m)} AS r,
{id: elementId(m), name: m.name} AS t
LIMIT $num
"""
fallback_results = tx.run(fallback_query, num=int(num))
formatted_results = {'nodes': [], 'edges': []}
for item in fallback_results:
formatted_results['nodes'].extend([item['h'], item['t']])
formatted_results['edges'].append(item['r'])
return formatted_results
except Exception as e:
# 如果连通查询失败,使用原始查询作为备选
logger.warning(f"Connected subgraph query failed, falling back to simple query: {e}")
fallback_query = """
MATCH (n:Entity)-[r]-(m:Entity)
WHERE elementId(n) < elementId(m)
RETURN
{id: elementId(n), name: n.name} AS h,
{type: r.type, source_id: elementId(n), target_id: elementId(m)} AS r,
{id: elementId(m), name: m.name} AS t
LIMIT $num
"""
results = tx.run(fallback_query, num=int(num))
formatted_results = {'nodes': [], 'edges': []}
for item in results:
formatted_results['nodes'].extend([item['h'], item['t']])
formatted_results['edges'].append(item['r'])
return formatted_results
2024-09-06 12:54:17 +08:00
with self.driver.session() as session:
results = session.execute_read(query, num)
return results
2024-09-06 12:54:17 +08:00
2024-07-15 18:58:34 +08:00
def create_graph_database(self, kgdb_name):
"""创建新的数据库,如果已存在则返回已有数据库的名称"""
assert self.driver is not None, "Database is not connected"
2024-07-15 18:58:34 +08:00
with self.driver.session() as session:
existing_databases = session.run("SHOW DATABASES")
existing_db_names = [db['name'] for db in existing_databases]
2024-07-16 18:14:27 +08:00
2024-07-15 18:58:34 +08:00
if existing_db_names:
print(f"已存在数据库: {existing_db_names[0]}")
return existing_db_names[0] # 返回所有已有数据库名称
2024-07-16 18:14:27 +08:00
session.run(f"CREATE DATABASE {kgdb_name}") # type: ignore
2024-07-15 18:58:34 +08:00
print(f"数据库 '{kgdb_name}' 创建成功.")
return kgdb_name # 返回创建的数据库名称
2024-07-16 18:14:27 +08:00
2024-09-14 02:44:53 +08:00
def use_database(self, kgdb_name="neo4j"):
2024-07-15 18:58:34 +08:00
"""切换到指定数据库"""
2024-09-14 02:44:53 +08:00
assert kgdb_name == self.kgdb_name, f"传入的数据库名称 '{kgdb_name}' 与当前实例的数据库名称 '{self.kgdb_name}' 不一致"
if self.status == "closed":
self.start()
2024-07-15 18:58:34 +08:00
2025-04-23 21:18:39 +08:00
async def txt_add_vector_entity(self, triples, kgdb_name='neo4j'):
2024-07-20 20:30:32 +08:00
"""添加实体三元组"""
assert self.driver is not None, "Database is not connected"
2024-07-20 20:30:32 +08:00
self.use_database(kgdb_name)
def _index_exists(tx, index_name):
2025-02-28 00:38:59 +08:00
"""检查索引是否存在"""
2024-07-20 20:30:32 +08:00
result = tx.run("SHOW INDEXES")
for record in result:
if record["name"] == index_name:
return True
return False
2025-02-28 00:38:59 +08:00
2024-07-20 20:30:32 +08:00
def _create_graph(tx, data):
2025-02-28 00:38:59 +08:00
"""添加一个三元组"""
2024-07-20 20:30:32 +08:00
for entry in data:
tx.run("""
MERGE (h:Entity:Upload {name: $h})
MERGE (t:Entity:Upload {name: $t})
2024-07-20 20:30:32 +08:00
MERGE (h)-[r:RELATION {type: $r}]->(t)
""", h=entry['h'], t=entry['t'], r=entry['r'])
2025-02-28 00:38:59 +08:00
2024-09-06 12:54:17 +08:00
def _create_vector_index(tx, dim):
2025-02-28 00:38:59 +08:00
"""创建向量索引"""
# NOTE 这里是否是会重复构建索引?
2024-09-06 12:54:17 +08:00
index_name = "entityEmbeddings"
2024-07-20 20:30:32 +08:00
if not _index_exists(tx, index_name):
tx.run(f"""
CREATE VECTOR INDEX {index_name}
FOR (n: Entity) ON (n.embedding)
OPTIONS {{indexConfig: {{
2024-09-06 12:54:17 +08:00
`vector.dimensions`: {dim},
2024-07-20 20:30:32 +08:00
`vector.similarity_function`: 'cosine'
}} }};
""")
2024-09-06 12:54:17 +08:00
def _get_nodes_without_embedding(tx, entity_names):
"""获取没有embedding的节点列表"""
# 构建参数字典,将列表转换为"param0"、"param1"等键值对形式
params = {f"param{i}": name for i, name in enumerate(entity_names)}
# 构建查询参数列表
param_placeholders = ", ".join([f"${key}" for key in params.keys()])
# 执行查询
result = tx.run(f"""
MATCH (n:Entity)
WHERE n.name IN [{param_placeholders}] AND n.embedding IS NULL
RETURN n.name AS name
""", params)
return [record["name"] for record in result]
2025-04-23 21:18:39 +08:00
def _batch_set_embeddings(tx, entity_embedding_pairs):
"""批量设置实体的嵌入向量"""
for entity_name, embedding in entity_embedding_pairs:
tx.run("""
MATCH (e:Entity {name: $name})
CALL db.create.setNodeVectorProperty(e, 'embedding', $embedding)
""", name=entity_name, embedding=embedding)
2025-02-28 00:38:59 +08:00
# 判断模型名称是否匹配
self.embed_model_name = self.embed_model_name or config.embed_model
cur_embed_info = config.embed_model_names.get(self.embed_model_name)
logger.warning(f"embed_model_name={self.embed_model_name}, {cur_embed_info=}")
assert self.embed_model_name == config.embed_model or self.embed_model_name is None, \
f"embed_model_name={self.embed_model_name}, {config.embed_model=}"
2025-02-28 00:38:59 +08:00
2024-07-20 20:30:32 +08:00
with self.driver.session() as session:
2025-02-28 00:38:59 +08:00
logger.info(f"Adding entity to {kgdb_name}")
2024-07-20 20:30:32 +08:00
session.execute_write(_create_graph, triples)
logger.info(f"Creating vector index for {kgdb_name} with {config.embed_model}")
2025-02-28 00:38:59 +08:00
session.execute_write(_create_vector_index, cur_embed_info['dimension'])
2025-04-23 21:18:39 +08:00
# 收集所有需要处理的实体名称,去重
all_entities = []
for entry in triples:
if entry['h'] not in all_entities:
all_entities.append(entry['h'])
if entry['t'] not in all_entities:
all_entities.append(entry['t'])
# 筛选出没有embedding的节点
nodes_without_embedding = session.execute_read(_get_nodes_without_embedding, all_entities)
if not nodes_without_embedding:
2025-05-24 11:29:45 +08:00
logger.info("所有实体已有embedding无需重新计算")
return
logger.info(f"需要为{len(nodes_without_embedding)}/{len(all_entities)}个实体计算embedding")
2025-04-23 21:18:39 +08:00
# 批量处理实体
max_batch_size = 1024 # 限制此部分的主要是内存大小 1024 * 1024 * 4 / 1024 / 1024 = 4GB
total_entities = len(nodes_without_embedding)
2025-04-23 21:18:39 +08:00
for i in range(0, total_entities, max_batch_size):
batch_entities = nodes_without_embedding[i:i+max_batch_size]
2025-05-24 11:29:45 +08:00
logger.debug(
f"Processing entities batch "
f"{i//max_batch_size + 1}/{(total_entities-1)//max_batch_size + 1} "
f"({len(batch_entities)} entities)"
)
2025-04-23 21:18:39 +08:00
# 批量获取嵌入向量
batch_embeddings = await self.aget_embedding(batch_entities)
# 将实体名称和嵌入向量配对
entity_embedding_pairs = list(zip(batch_entities, batch_embeddings))
# 批量写入数据库
session.execute_write(_batch_set_embeddings, entity_embedding_pairs)
2024-07-20 20:30:32 +08:00
2025-02-28 02:40:34 +08:00
# 数据添加完成后保存图信息
self.save_graph_info()
2025-04-23 21:18:39 +08:00
async def jsonl_file_add_entity(self, file_path, kgdb_name='neo4j'):
assert self.driver is not None, "Database is not connected"
2024-07-24 18:32:14 +08:00
self.status = "processing"
2024-10-14 16:51:20 +08:00
kgdb_name = kgdb_name or 'neo4j'
2024-07-20 20:30:32 +08:00
self.use_database(kgdb_name) # 切换到指定数据库
2025-02-28 00:38:59 +08:00
logger.info(f"Start adding entity to {kgdb_name} with {file_path}")
2024-07-24 18:32:14 +08:00
2024-07-20 20:30:32 +08:00
def read_triples(file_path):
2025-05-24 11:29:45 +08:00
with open(file_path, encoding='utf-8') as file:
2024-07-20 20:30:32 +08:00
for line in file:
2025-03-30 11:09:46 +08:00
if line.strip():
yield json.loads(line.strip())
triples = list(read_triples(file_path))
2025-04-23 21:18:39 +08:00
await self.txt_add_vector_entity(triples, kgdb_name)
2024-09-06 12:54:17 +08:00
2024-07-24 18:32:14 +08:00
self.status = "open"
2025-02-28 02:40:34 +08:00
# 更新并保存图数据库信息
self.save_graph_info()
2024-07-20 20:30:32 +08:00
return kgdb_name
2024-07-15 18:58:34 +08:00
def delete_entity(self, entity_name=None, kgdb_name="neo4j"):
"""删除数据库中的指定实体三元组, 参数entity_name为空则删除全部实体"""
assert self.driver is not None, "Database is not connected"
2024-07-15 18:58:34 +08:00
self.use_database(kgdb_name)
with self.driver.session() as session:
if entity_name:
session.execute_write(self._delete_specific_entity, entity_name)
else:
session.execute_write(self._delete_all_entities)
def _delete_specific_entity(self, tx, entity_name):
query = """
MATCH (n {name: $entity_name})
DETACH DELETE n
"""
tx.run(query, entity_name=entity_name)
def _delete_all_entities(self, tx):
query = """
MATCH (n)
DETACH DELETE n
"""
tx.run(query)
def query_node(self, entity_name, threshold=0.9, kgdb_name='neo4j', hops=2, max_entities=5, return_format='graph', **kwargs):
2025-05-07 00:38:33 +08:00
"""知识图谱查询节点的入口:"""
assert self.driver is not None, "Database is not connected"
assert self.is_running(), "图数据库未启动"
2025-02-28 02:40:34 +08:00
2024-07-15 18:58:34 +08:00
self.use_database(kgdb_name)
# 使用向量索引进行查询
results_sim = self._query_with_vector_sim(entity_name, kgdb_name, threshold)
results_fuzzy = self._query_with_fuzzy_match(entity_name, kgdb_name)
results = results_sim + results_fuzzy
qualified_entities = [result[0] for result in results][:max_entities]
logger.debug(f"Graph Query Entities: {entity_name}, {qualified_entities=}")
# 对每个合格的实体进行查询
all_query_results = {'nodes': [], 'edges': [], 'triples': []}
for entity in qualified_entities:
query_result = self._query_specific_entity(entity_name=entity, kgdb_name=kgdb_name, hops=hops)
if return_format == 'graph':
all_query_results['nodes'].extend(query_result['nodes'])
all_query_results['edges'].extend(query_result['edges'])
elif return_format == 'triples':
all_query_results['triples'].extend(query_result['triples'])
else:
raise ValueError(f"Invalid return_format: {return_format}")
return all_query_results
def _query_with_fuzzy_match(self, keyword, kgdb_name='neo4j'):
"""模糊查询"""
assert self.driver is not None, "Database is not connected"
self.use_database(kgdb_name)
def query_fuzzy_match(tx, keyword):
result = tx.run("""
MATCH (n:Entity)
WHERE n.name CONTAINS $keyword
RETURN DISTINCT n.name AS name
""", keyword=keyword)
values = result.values()
logger.debug(f"Fuzzy Query Results: {values}")
return values
with self.driver.session() as session:
return session.execute_read(query_fuzzy_match, keyword)
def _query_with_vector_sim(self, keyword, kgdb_name='neo4j', threshold=0.9):
"""向量查询"""
assert self.driver is not None, "Database is not connected"
self.use_database(kgdb_name)
def _index_exists(tx, index_name):
"""检查索引是否存在"""
result = tx.run("SHOW INDEXES")
for record in result:
if record["name"] == index_name:
return True
return False
def query_by_vector(tx, text, threshold):
# 首先检查索引是否存在
if not _index_exists(tx, "entityEmbeddings"):
raise Exception("向量索引不存在,请先创建索引,或当前图谱中未上传任何三元组(知识库中自动构建的,不会在此处展示和检索)。")
2024-09-14 02:44:53 +08:00
embedding = self.get_embedding(text)
result = tx.run("""
CALL db.index.vector.queryNodes('entityEmbeddings', 10, $embedding)
YIELD node AS similarEntity, score
RETURN similarEntity.name AS name, score
""", embedding=embedding)
return [r for r in result if r["score"] > threshold]
2025-02-28 00:38:59 +08:00
with self.driver.session() as session:
results = session.execute_read(query_by_vector, keyword, threshold=threshold)
return results
2025-02-28 00:38:59 +08:00
2024-07-15 18:58:34 +08:00
def _query_specific_entity(self, entity_name, kgdb_name='neo4j', hops=2, limit=100):
2025-02-28 00:38:59 +08:00
"""查询指定实体三元组信息(无向关系)"""
assert self.driver is not None, "Database is not connected"
2025-04-13 21:06:23 +08:00
if not entity_name:
logger.warning("实体名称为空")
return []
2024-07-15 18:58:34 +08:00
self.use_database(kgdb_name)
2025-04-13 21:06:23 +08:00
def query(tx, entity_name, hops, limit):
try:
query_str = """
MATCH (n {name: $entity_name})-[r1]->(m1)
RETURN
{id: elementId(n), name: n.name} AS h,
{type: r1.type, source_id: elementId(n), target_id: elementId(m1)} AS r,
{id: elementId(m1), name: m1.name} AS t
UNION
MATCH (n {name: $entity_name})-[r1]->(m1)-[r2]->(m2)
RETURN
{id: elementId(m1), name: m1.name} AS h,
{type: r2.type, source_id: elementId(m1), target_id: elementId(m2)} AS r,
{id: elementId(m2), name: m2.name} AS t
UNION
MATCH (m1)-[r1]->(n {name: $entity_name})
RETURN
{id: elementId(m1), name: m1.name} AS h,
{type: r1.type, source_id: elementId(m1), target_id: elementId(n)} AS r,
{id: elementId(n), name: n.name} AS t
UNION
MATCH (m2)-[r2]->(m1)-[r1]->(n {name: $entity_name})
RETURN
{id: elementId(m2), name: m2.name} AS h,
{type: r2.type, source_id: elementId(m2), target_id: elementId(m1)} AS r,
{id: elementId(m1), name: m1.name} AS t
2025-04-13 21:06:23 +08:00
LIMIT $limit
"""
results = tx.run(query_str, entity_name=entity_name, limit=limit)
2025-04-13 21:06:23 +08:00
if not results:
2025-04-13 21:06:23 +08:00
logger.info(f"未找到实体 {entity_name} 的相关信息")
return {}
formatted_results = {'nodes': [], 'edges': [], 'triples': []}
for item in results:
formatted_results['nodes'].extend([item['h'], item['t']])
formatted_results['edges'].append(item['r'])
formatted_results['triples'].append((item['h']['name'], item['r']['type'], item['t']['name']))
2025-04-13 21:06:23 +08:00
logger.debug(f"Query Results: {results}")
return formatted_results
2025-04-13 21:06:23 +08:00
except Exception as e:
logger.error(f"查询实体 {entity_name} 失败: {str(e)}")
return []
try:
with self.driver.session() as session:
return session.execute_read(query, entity_name, hops, limit)
2025-04-13 21:06:23 +08:00
except Exception as e:
logger.error(f"数据库会话异常: {str(e)}")
return []
2024-07-15 18:58:34 +08:00
2025-04-23 21:18:39 +08:00
async def aget_embedding(self, text):
if isinstance(text, list):
outputs = await self.embed_model.abatch_encode(text, batch_size=40)
2025-04-23 21:18:39 +08:00
return outputs
else:
outputs = await self.embed_model.aencode(text)
2025-04-23 21:18:39 +08:00
return outputs
2024-07-20 20:30:32 +08:00
def get_embedding(self, text):
2025-04-23 21:18:39 +08:00
if isinstance(text, list):
outputs = self.embed_model.batch_encode(text, batch_size=40)
2025-04-23 21:18:39 +08:00
return outputs
else:
outputs = self.embed_model.encode([text])[0]
2025-02-28 00:38:59 +08:00
return outputs
2024-07-24 18:32:14 +08:00
2024-07-20 20:30:32 +08:00
def set_embedding(self, tx, entity_name, embedding):
tx.run("""
MATCH (e:Entity {name: $name})
CALL db.create.setNodeVectorProperty(e, 'embedding', $embedding)
""", name=entity_name, embedding=embedding)
2024-07-24 18:32:14 +08:00
def get_graph_info(self, graph_name="neo4j"):
assert self.driver is not None, "Database is not connected"
self.use_database(graph_name)
def query(tx):
# 只统计包含Entity标签的节点
entity_count = tx.run("MATCH (n:Entity) RETURN count(n) AS count").single()["count"]
# 只统计包含RELATION标签的关系
relationship_count = tx.run("MATCH ()-[r:RELATION]->() RETURN count(r) AS count").single()["count"]
triples_count = tx.run("MATCH (n:Entity)-[r:RELATION]->(m:Entity) RETURN count(n) AS count").single()["count"]
# 获取所有标签
labels = tx.run("CALL db.labels() YIELD label RETURN collect(label) AS labels").single()["labels"]
return {
"graph_name": graph_name,
"entity_count": entity_count,
"relationship_count": relationship_count,
"triples_count": triples_count,
"labels": labels,
2025-02-28 02:40:34 +08:00
"status": self.status,
"embed_model_name": self.embed_model_name,
"unindexed_node_count": self.query_nodes_without_embedding(graph_name)
2025-02-28 02:40:34 +08:00
}
try:
if self.is_running():
# 获取数据库信息
with self.driver.session() as session:
graph_info = session.execute_read(query)
2025-04-01 18:37:30 +08:00
# 添加时间戳
from datetime import datetime
graph_info["last_updated"] = datetime.now().isoformat()
return graph_info
else:
logger.warning(f"图数据库未连接或未运行:{self.status=}")
return None
except Exception as e:
logger.error(f"获取图数据库信息失败:{e}, {traceback.format_exc()}")
return None
def save_graph_info(self, graph_name="neo4j"):
"""
将图数据库的基本信息保存到工作目录中的JSON文件
保存的信息包括数据库名称状态嵌入模型名称等
"""
try:
graph_info = self.get_graph_info(graph_name)
if graph_info is None:
2025-05-24 11:29:45 +08:00
logger.error("图数据库信息为空,无法保存")
return False
2025-02-28 02:40:34 +08:00
info_file_path = os.path.join(self.work_dir, "graph_info.json")
with open(info_file_path, 'w', encoding='utf-8') as f:
json.dump(graph_info, f, ensure_ascii=False, indent=2)
2025-04-14 00:15:30 +08:00
# logger.info(f"图数据库信息已保存到:{info_file_path}")
2025-02-28 02:40:34 +08:00
return True
except Exception as e:
logger.error(f"保存图数据库信息失败:{e}")
return False
2025-03-08 22:49:13 +08:00
def query_nodes_without_embedding(self, kgdb_name='neo4j'):
"""查询没有嵌入向量的节点
Returns:
list: 没有嵌入向量的节点列表
"""
assert self.driver is not None, "Database is not connected"
2025-03-08 22:49:13 +08:00
self.use_database(kgdb_name)
def query(tx):
result = tx.run("""
MATCH (n:Entity)
WHERE n.embedding IS NULL
RETURN n.name AS name
""")
return [record["name"] for record in result]
with self.driver.session() as session:
return session.execute_read(query)
2025-02-28 02:40:34 +08:00
def load_graph_info(self):
"""
从工作目录中的JSON文件加载图数据库的基本信息
返回True表示加载成功False表示加载失败
"""
try:
info_file_path = os.path.join(self.work_dir, "graph_info.json")
if not os.path.exists(info_file_path):
2025-04-28 22:53:13 +08:00
logger.debug(f"图数据库信息文件不存在:{info_file_path}")
2025-02-28 02:40:34 +08:00
return False
2025-05-24 11:29:45 +08:00
with open(info_file_path, encoding='utf-8') as f:
2025-02-28 02:40:34 +08:00
graph_info = json.load(f)
# 更新对象属性
if graph_info.get("embed_model_name"):
self.embed_model_name = graph_info["embed_model_name"]
# 如果需要,可以加载更多信息
# 注意这里不更新self.kgdb_name因为它是在初始化时设置的
logger.info(f"已加载图数据库信息,最后更新时间:{graph_info.get('last_updated')}")
return True
except Exception as e:
logger.error(f"加载图数据库信息失败:{e}")
return False
2025-03-08 22:49:13 +08:00
def add_embedding_to_nodes(self, node_names=None, kgdb_name='neo4j'):
"""为节点添加嵌入向量
Args:
node_names (list, optional): 要添加嵌入向量的节点名称列表None表示所有没有嵌入向量的节点
kgdb_name (str, optional): 图数据库名称默认为'neo4j'
Returns:
int: 成功添加嵌入向量的节点数量
"""
assert self.driver is not None, "Database is not connected"
2025-03-08 22:49:13 +08:00
self.use_database(kgdb_name)
# 如果node_names为None则获取所有没有嵌入向量的节点
if node_names is None:
node_names = self.query_nodes_without_embedding(kgdb_name)
count = 0
with self.driver.session() as session:
for node_name in node_names:
try:
embedding = self.get_embedding(node_name)
session.execute_write(self.set_embedding, node_name, embedding)
count += 1
except Exception as e:
2025-03-19 01:21:51 +08:00
logger.error(f"为节点 '{node_name}' 添加嵌入向量失败: {e}, {traceback.format_exc()}")
2025-03-08 22:49:13 +08:00
return count
def format_general_results(self, results):
nodes = []
edges = []
for item in results:
nodes.extend([item['h'], item['t']])
edges.append(item['r'])
formatted_results = {"nodes": nodes, "edges": edges}
return formatted_results
def _extract_relationship_info(self, relationship, source_name=None, target_name=None, node_dict=None):
"""
提取关系信息并返回格式化的节点和边信息
"""
rel_id = relationship.element_id
nodes = relationship.nodes
if len(nodes) != 2:
return None, None
source, target = nodes
source_id = source.element_id
target_id = target.element_id
# 如果没有提供 source_name 或 target_name则需要 node_dict
if source_name is None or target_name is None:
assert node_dict is not None, "node_dict is required when source_name or target_name is None"
source_name = node_dict[source_id]["name"] if source_name is None else source_name
target_name = node_dict[target_id]["name"] if target_name is None else target_name
relationship_type = relationship._properties.get("type", "unknown")
if relationship_type == "unknown":
relationship_type = relationship.type
edge_info = {
"id": rel_id,
"type": relationship_type,
"source_id": source_id,
"target_id": target_id,
"source_name": source_name,
"target_name": target_name,
}
node_info = [
{"id": source_id, "name": source_name},
{"id": target_id, "name": target_name},
]
return node_info, edge_info
2025-04-13 21:06:23 +08:00
def clean_triples_embedding(triples):
for item in triples:
if hasattr(item[0], '_properties'):
item[0]._properties['embedding'] = None
if hasattr(item[2], '_properties'):
item[2]._properties['embedding'] = None
return triples
2024-07-15 18:58:34 +08:00
if __name__ == "__main__":
2025-03-30 11:09:46 +08:00
pass