style: ruff format

This commit is contained in:
Wenjie Zhang 2025-10-24 00:11:52 +08:00
parent 92a110d955
commit a1fdcbc36c
17 changed files with 115 additions and 107 deletions

View File

@ -27,6 +27,7 @@ lint:
format: format:
uv run ruff format . uv run ruff format .
uv run ruff check . --fix uv run ruff check . --fix
uv run python -m ruff check --select I src --fix
format_diff: format_diff:
uv run ruff format --diff . uv run ruff format --diff .

View File

@ -112,7 +112,8 @@ async def get_agent(current_user: User = Depends(get_required_user)):
metadata = yaml.safe_load(f) metadata = yaml.safe_load(f)
return {"agents": agents, "metadata": metadata} return {"agents": agents, "metadata": metadata}
#TODO:[未完成]这个thread_id在前端是直接生成的1234最好传入thread_id时做校验只允许uuid4
# TODO:[未完成]这个thread_id在前端是直接生成的1234最好传入thread_id时做校验只允许uuid4
@chat.post("/agent/{agent_id}") @chat.post("/agent/{agent_id}")
async def chat_agent( async def chat_agent(
agent_id: str, agent_id: str,
@ -254,7 +255,8 @@ async def chat_agent(
logger.error(f"Error saving messages from LangGraph state: {e}") logger.error(f"Error saving messages from LangGraph state: {e}")
logger.error(traceback.format_exc()) logger.error(traceback.format_exc())
#TODO:[功能建议]针对需要人工审批后再执行的工具可以使用langgraph的interrupt方法中断对话等待用户输入后再使用command跳转回去 # TODO:[功能建议]针对需要人工审批后再执行的工具,
# 可以使用langgraph的interrupt方法中断对话等待用户输入后再使用command跳转回去
async def stream_messages(): async def stream_messages():
# 代表服务端已经收到了请求 # 代表服务端已经收到了请求
yield make_chunk(status="init", meta=meta, msg=HumanMessage(content=query).model_dump()) yield make_chunk(status="init", meta=meta, msg=HumanMessage(content=query).model_dump())

View File

@ -1,11 +1,11 @@
import asyncio
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from fastapi import FastAPI from fastapi import FastAPI
from server.services import tasker from server.services import tasker
#TODO:[已完成]使用lifespan进行统一生命周期管理 # TODO:[已完成]使用lifespan进行统一生命周期管理
@asynccontextmanager @asynccontextmanager
async def lifespan(app: FastAPI): async def lifespan(app: FastAPI):

View File

@ -1,9 +1,11 @@
from threading import Lock from threading import Lock
class SingletonMeta(type): class SingletonMeta(type):
""" """
This is a thread-safe implementation of Singleton. This is a thread-safe implementation of Singleton.
""" """
_instances = {} _instances = {}
_lock: Lock = Lock() _lock: Lock = Lock()
@ -12,4 +14,4 @@ class SingletonMeta(type):
if cls not in cls._instances: if cls not in cls._instances:
instance = super().__call__(*args, **kwargs) instance = super().__call__(*args, **kwargs)
cls._instances[cls] = instance cls._instances[cls] = instance
return cls._instances[cls] return cls._instances[cls]

View File

@ -9,8 +9,9 @@ from src.agents.common.tools import get_buildin_tools
from src.storage.minio import upload_image_to_minio from src.storage.minio import upload_image_to_minio
from src.utils import logger from src.utils import logger
#TODO:[已完成]修改了tool定义的示例使用更符合langgraph调用的方式
@tool(name_or_callable="全能计算器",description="可以对给定的2个数字选择进行加减乘除四种计算") # TODO:[已完成]修改了tool定义的示例使用更符合langgraph调用的方式
@tool(name_or_callable="全能计算器", description="可以对给定的2个数字选择进行加减乘除四种计算")
def calculator(a: float, b: float, operation: str) -> float: def calculator(a: float, b: float, operation: str) -> float:
""" """
可以对给定的2个数字选择进行加减乘除四种计算 可以对给定的2个数字选择进行加减乘除四种计算

View File

@ -1,12 +1,12 @@
from __future__ import annotations from __future__ import annotations
import os import os
from pathlib import Path
from abc import abstractmethod from abc import abstractmethod
from pathlib import Path
from langgraph.graph.state import CompiledStateGraph
from langgraph.checkpoint.memory import InMemorySaver from langgraph.checkpoint.memory import InMemorySaver
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver, aiosqlite from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver, aiosqlite
from langgraph.graph.state import CompiledStateGraph
from src import config as sys_config from src import config as sys_config
from src.agents.common.context import BaseContext from src.agents.common.context import BaseContext

View File

@ -11,7 +11,7 @@ from src import config, graph_base, knowledge_base
from src.utils import logger from src.utils import logger
@tool @tool(name_or_callable="查询知识图谱", description="使用这个工具可以查询知识图谱中包含的三元组信息。")
def query_knowledge_graph(query: Annotated[str, "The keyword to query knowledge graph."]) -> Any: def query_knowledge_graph(query: Annotated[str, "The keyword to query knowledge graph."]) -> Any:
"""Use this to query knowledge graph, which include some food domain knowledge.""" """Use this to query knowledge graph, which include some food domain knowledge."""
try: try:
@ -35,7 +35,9 @@ def get_static_tools() -> list:
# 检查是否启用网页搜索 # 检查是否启用网页搜索
if config.enable_web_search: if config.enable_web_search:
static_tools.append(TavilySearch(max_results=10)) search = TavilySearch(max_results=10)
search.metadata = {"name": "Tavily 网页搜索"}
static_tools.append(search)
return static_tools return static_tools
@ -76,9 +78,6 @@ def get_kb_based_tools() -> list:
for db_id, retrieve_info in retrievers.items(): for db_id, retrieve_info in retrievers.items():
try: try:
# 使用改进的工具ID生成策略
tool_id = f"query_{db_id[:8]}"
# 构建工具描述 # 构建工具描述
description = ( description = (
f"使用 {retrieve_info['name']} 知识库进行检索。\n" f"使用 {retrieve_info['name']} 知识库进行检索。\n"
@ -88,10 +87,12 @@ def get_kb_based_tools() -> list:
# 使用工厂函数创建检索器包装函数,避免闭包问题 # 使用工厂函数创建检索器包装函数,避免闭包问题
retriever_wrapper = _create_retriever_wrapper(db_id, retrieve_info) retriever_wrapper = _create_retriever_wrapper(db_id, retrieve_info)
safename = retrieve_info["name"].replace(" ", "_")[:20]
# 使用 StructuredTool.from_function 创建异步工具 # 使用 StructuredTool.from_function 创建异步工具
tool = StructuredTool.from_function( tool = StructuredTool.from_function(
coroutine=retriever_wrapper, coroutine=retriever_wrapper,
name=tool_id, name=safename,
description=description, description=description,
args_schema=KnowledgeRetrieverModel, args_schema=KnowledgeRetrieverModel,
metadata=retrieve_info["metadata"] | {"tag": ["knowledgebase"]}, metadata=retrieve_info["metadata"] | {"tag": ["knowledgebase"]},

View File

@ -13,7 +13,7 @@ from typing import Any
import tomli import tomli
import tomli_w import tomli_w
from pydantic import BaseModel, Field, field_validator from pydantic import BaseModel, Field
from src.config.static.models import ( from src.config.static.models import (
DEFAULT_CHAT_MODEL_PROVIDERS, DEFAULT_CHAT_MODEL_PROVIDERS,
@ -172,13 +172,10 @@ class Config(BaseModel):
self.model_dir = os.environ.get("MODEL_DIR", self.model_dir) self.model_dir = os.environ.get("MODEL_DIR", self.model_dir)
if self.model_dir: if self.model_dir:
if os.path.exists(self.model_dir): if os.path.exists(self.model_dir):
logger.debug( logger.debug(f"Model directory ({self.model_dir}) contains: {os.listdir(self.model_dir)}")
f"Model directory ({self.model_dir}) contains: {os.listdir(self.model_dir)}"
)
else: else:
logger.warning( logger.warning(
f"Model directory ({self.model_dir}) does not exist. " f"Model directory ({self.model_dir}) does not exist. If not configured, please ignore it."
"If not configured, please ignore it."
) )
# 检查模型提供商的环境变量 # 检查模型提供商的环境变量
@ -195,14 +192,10 @@ class Config(BaseModel):
self.enable_web_search = True self.enable_web_search = True
# 获取可用的模型提供商 # 获取可用的模型提供商
self.valuable_model_provider = [ self.valuable_model_provider = [k for k, v in self.model_provider_status.items() if v]
k for k, v in self.model_provider_status.items() if v
]
if not self.valuable_model_provider: if not self.valuable_model_provider:
raise ValueError( raise ValueError("No model provider available, please check your `.env` file.")
"No model provider available, please check your `.env` file."
)
def save(self): def save(self):
"""保存配置到 TOML 文件(仅保存用户修改的字段)""" """保存配置到 TOML 文件(仅保存用户修改的字段)"""
@ -251,15 +244,11 @@ class Config(BaseModel):
) )
# 添加模型信息(转换为字典格式供前端使用) # 添加模型信息(转换为字典格式供前端使用)
config_dict["model_names"] = { config_dict["model_names"] = {provider: info.model_dump() for provider, info in self.model_names.items()}
provider: info.model_dump() for provider, info in self.model_names.items()
}
config_dict["embed_model_names"] = { config_dict["embed_model_names"] = {
model_id: info.model_dump() for model_id, info in self.embed_model_names.items() model_id: info.model_dump() for model_id, info in self.embed_model_names.items()
} }
config_dict["reranker_names"] = { config_dict["reranker_names"] = {model_id: info.model_dump() for model_id, info in self.reranker_names.items()}
model_id: info.model_dump() for model_id, info in self.reranker_names.items()
}
# 添加运行时状态信息 # 添加运行时状态信息
config_dict["model_provider_status"] = self.model_provider_status config_dict["model_provider_status"] = self.model_provider_status
@ -269,10 +258,12 @@ class Config(BaseModel):
for field_name, field_info in Config.model_fields.items(): for field_name, field_info in Config.model_fields.items():
if not field_info.exclude: # 排除内部字段 if not field_info.exclude: # 排除内部字段
fields_info[field_name] = { fields_info[field_name] = {
'des': field_info.description, "des": field_info.description,
'default': field_info.default, "default": field_info.default,
'type': field_info.annotation.__name__ if hasattr(field_info.annotation, '__name__') else str(field_info.annotation), "type": field_info.annotation.__name__
'exclude': field_info.exclude if hasattr(field_info, 'exclude') else False, if hasattr(field_info.annotation, "__name__")
else str(field_info.annotation),
"exclude": field_info.exclude if hasattr(field_info, "exclude") else False,
} }
config_dict["_config_items"] = fields_info config_dict["_config_items"] = fields_info
@ -301,14 +292,12 @@ class Config(BaseModel):
def __getitem__(self, key: str) -> Any: def __getitem__(self, key: str) -> Any:
"""支持字典式访问 config[key]""" """支持字典式访问 config[key]"""
logger.warning("Using deprecated dict-style access for Config. " logger.warning("Using deprecated dict-style access for Config. Please use attribute access instead.")
"Please use attribute access instead.")
return getattr(self, key, None) return getattr(self, key, None)
def __setitem__(self, key: str, value: Any): def __setitem__(self, key: str, value: Any):
"""支持字典式赋值 config[key] = value""" """支持字典式赋值 config[key] = value"""
logger.warning("Using deprecated dict-style assignment for Config. " logger.warning("Using deprecated dict-style assignment for Config. Please use attribute access instead.")
"Please use attribute access instead.")
setattr(self, key, value) setattr(self, key, value)
def update(self, other: dict): def update(self, other: dict):
@ -352,8 +341,7 @@ class Config(BaseModel):
else: else:
# 保存所有 model_names # 保存所有 model_names
user_config["model_names"] = { user_config["model_names"] = {
provider: info.model_dump() provider: info.model_dump() for provider, info in self.model_names.items()
for provider, info in self.model_names.items()
} }
# 记录整个 model_names 字段的修改 # 记录整个 model_names 字段的修改
self._user_modified_fields.add("model_names") self._user_modified_fields.add("model_names")

View File

@ -1,7 +1,7 @@
import json import json
import os import os
import tempfile
import shutil import shutil
import tempfile
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import Any from typing import Any
@ -144,8 +144,8 @@ class KnowledgeBase(ABC):
"name": database_name, "name": database_name,
"description": description, "description": description,
"kb_type": self.kb_type, "kb_type": self.kb_type,
"embed_info": embed_info.model_dump() if hasattr(embed_info, 'model_dump') else embed_info, "embed_info": embed_info.model_dump() if hasattr(embed_info, "model_dump") else embed_info,
"llm_info": llm_info.model_dump() if hasattr(llm_info, 'model_dump') else llm_info, "llm_info": llm_info.model_dump() if hasattr(llm_info, "model_dump") else llm_info,
"metadata": kwargs, "metadata": kwargs,
"created_at": utc_isoformat(), "created_at": utc_isoformat(),
} }
@ -559,7 +559,7 @@ class KnowledgeBase(ABC):
def _serialize_metadata(self, obj): def _serialize_metadata(self, obj):
"""递归序列化元数据中的 Pydantic 模型""" """递归序列化元数据中的 Pydantic 模型"""
if hasattr(obj, 'dict'): if hasattr(obj, "dict"):
return obj.dict() return obj.dict()
elif isinstance(obj, dict): elif isinstance(obj, dict):
return {k: self._serialize_metadata(v) for k, v in obj.items()} return {k: self._serialize_metadata(v) for k, v in obj.items()}
@ -589,8 +589,7 @@ class KnowledgeBase(ABC):
# 原子性写入(使用临时文件) # 原子性写入(使用临时文件)
with tempfile.NamedTemporaryFile( with tempfile.NamedTemporaryFile(
mode='w', dir=os.path.dirname(meta_file), mode="w", dir=os.path.dirname(meta_file), prefix=".tmp_", suffix=".json", delete=False
prefix='.tmp_', suffix='.json', delete=False
) as tmp_file: ) as tmp_file:
json.dump(data, tmp_file, ensure_ascii=False, indent=2) json.dump(data, tmp_file, ensure_ascii=False, indent=2)
temp_path = tmp_file.name temp_path = tmp_file.name

View File

@ -299,7 +299,7 @@ class GraphDatabase:
logger.info(f"Adding entity to {kgdb_name}") logger.info(f"Adding entity to {kgdb_name}")
session.execute_write(_create_graph, triples) session.execute_write(_create_graph, triples)
logger.info(f"Creating vector index for {kgdb_name} with {config.embed_model}") logger.info(f"Creating vector index for {kgdb_name} with {config.embed_model}")
session.execute_write(_create_vector_index, getattr(cur_embed_info, 'dimension', 1024)) session.execute_write(_create_vector_index, getattr(cur_embed_info, "dimension", 1024))
# 收集所有需要处理的实体名称,去重 # 收集所有需要处理的实体名称,去重
all_entities = [] all_entities = []

View File

@ -72,11 +72,11 @@ class ChromaKB(KnowledgeBase):
logger.info(f"Retrieved existing collection: {collection_name}") logger.info(f"Retrieved existing collection: {collection_name}")
# 检查现有集合的配置是否匹配当前的 embed_info # 检查现有集合的配置是否匹配当前的 embed_info
expected_model = getattr(embed_info, 'name', None) if embed_info else None expected_model = getattr(embed_info, "name", None) if embed_info else None
if expected_model is None and hasattr(embed_info, 'get'): if expected_model is None and hasattr(embed_info, "get"):
expected_model = embed_info.get('name') expected_model = embed_info.get("name")
elif embed_info and isinstance(embed_info, dict): elif embed_info and isinstance(embed_info, dict):
expected_model = embed_info.get('name') expected_model = embed_info.get("name")
expected_model = expected_model or "default" expected_model = expected_model or "default"
collection_metadata = collection.metadata or {} collection_metadata = collection.metadata or {}
current_model = collection_metadata.get("embedding_model", "unknown") current_model = collection_metadata.get("embedding_model", "unknown")
@ -93,13 +93,13 @@ class ChromaKB(KnowledgeBase):
except Exception: except Exception:
# 创建新集合 # 创建新集合
model_name = getattr(embed_info, 'name', None) if embed_info else None model_name = getattr(embed_info, "name", None) if embed_info else None
if model_name is None and hasattr(embed_info, 'get'): if model_name is None and hasattr(embed_info, "get"):
model_name = embed_info.get('name') model_name = embed_info.get("name")
elif embed_info and isinstance(embed_info, dict): elif embed_info and isinstance(embed_info, dict):
model_name = embed_info.get('name') model_name = embed_info.get("name")
model_name = model_name or 'default' model_name = model_name or "default"
logger.info(f"Creating new collection with embedding model: {model_name}") logger.info(f"Creating new collection with embedding model: {model_name}")
collection_metadata = { collection_metadata = {
"db_id": db_id, "db_id": db_id,

View File

@ -103,7 +103,7 @@ class MilvusKB(KnowledgeBase):
# 检查嵌入模型是否匹配 # 检查嵌入模型是否匹配
description = collection.description description = collection.description
expected_model = getattr(embed_info, 'name', 'default') if embed_info else "default" expected_model = getattr(embed_info, "name", "default") if embed_info else "default"
if expected_model not in description: if expected_model not in description:
logger.warning(f"Collection {collection_name} model mismatch, recreating...") logger.warning(f"Collection {collection_name} model mismatch, recreating...")
@ -116,8 +116,8 @@ class MilvusKB(KnowledgeBase):
except Exception: except Exception:
# 创建新集合 # 创建新集合
embedding_dim = getattr(embed_info, 'dimension', 1024) if embed_info else 1024 embedding_dim = getattr(embed_info, "dimension", 1024) if embed_info else 1024
model_name = getattr(embed_info, 'name', 'default') if embed_info else "default" model_name = getattr(embed_info, "name", "default") if embed_info else "default"
# 定义集合Schema # 定义集合Schema
fields = [ fields = [

View File

@ -2,7 +2,6 @@ import asyncio
import os import os
from pathlib import Path from pathlib import Path
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import ( from langchain_community.document_loaders import (
CSVLoader, CSVLoader,
JSONLoader, JSONLoader,
@ -12,10 +11,10 @@ from langchain_community.document_loaders import (
UnstructuredMarkdownLoader, UnstructuredMarkdownLoader,
UnstructuredWordDocumentLoader, UnstructuredWordDocumentLoader,
) )
from langchain_text_splitters import RecursiveCharacterTextSplitter
from src.utils import logger from src.utils import logger
SUPPORTED_FILE_EXTENSIONS: tuple[str, ...] = ( SUPPORTED_FILE_EXTENSIONS: tuple[str, ...] = (
".txt", ".txt",
".md", ".md",

View File

@ -104,16 +104,11 @@ class KnowledgeBaseManager:
shutil.copy2(meta_file, backup_file) shutil.copy2(meta_file, backup_file)
# 准备数据 # 准备数据
data = { data = {"databases": self.global_databases_meta, "updated_at": utc_isoformat(), "version": "2.0"}
"databases": self.global_databases_meta,
"updated_at": utc_isoformat(),
"version": "2.0"
}
# 原子性写入(使用临时文件) # 原子性写入(使用临时文件)
with tempfile.NamedTemporaryFile( with tempfile.NamedTemporaryFile(
mode='w', dir=os.path.dirname(meta_file), mode="w", dir=os.path.dirname(meta_file), prefix=".tmp_", suffix=".json", delete=False
prefix='.tmp_', suffix='.json', delete=False
) as tmp_file: ) as tmp_file:
json.dump(data, tmp_file, ensure_ascii=False, indent=2) json.dump(data, tmp_file, ensure_ascii=False, indent=2)
temp_path = tmp_file.name temp_path = tmp_file.name
@ -520,7 +515,7 @@ class KnowledgeBaseManager:
"chroma": {"missing_collections": [], "missing_files": []}, "chroma": {"missing_collections": [], "missing_files": []},
"milvus": {"missing_collections": [], "missing_files": []}, "milvus": {"missing_collections": [], "missing_files": []},
"total_missing_collections": 0, "total_missing_collections": 0,
"total_missing_files": 0 "total_missing_files": 0,
} }
logger.info("开始检测向量数据库与元数据的一致性...") logger.info("开始检测向量数据库与元数据的一致性...")
@ -573,10 +568,7 @@ class KnowledgeBaseManager:
if not collection_name.startswith("kb_"): if not collection_name.startswith("kb_"):
continue continue
collection_info = { collection_info = {"collection_name": collection_name, "detected_at": utc_isoformat()}
"collection_name": collection_name,
"detected_at": utc_isoformat()
}
# 尝试获取集合的基本信息 # 尝试获取集合的基本信息
try: try:
@ -588,7 +580,10 @@ class KnowledgeBaseManager:
collection_info["count"] = "unknown" collection_info["count"] = "unknown"
inconsistencies["missing_collections"].append(collection_info) inconsistencies["missing_collections"].append(collection_info)
logger.warning(f"发现 ChromaDB 中存在但 metadata 中缺失的集合: {collection_name} (文档数: {collection_info['count']})") logger.warning(
f"发现 ChromaDB 中存在但 metadata 中缺失的集合: {collection_name} "
f"(文档数: {collection_info['count']})"
)
# 检查文件级别的不一致(针对已知的数据库) # 检查文件级别的不一致(针对已知的数据库)
for db_id in metadata_collection_names: for db_id in metadata_collection_names:
@ -597,18 +592,23 @@ class KnowledgeBaseManager:
actual_count = collection.count() actual_count = collection.count()
# 获取 metadata 中记录的文件数量 # 获取 metadata 中记录的文件数量
metadata_files_count = sum(1 for file_info in chroma_kb.files_meta.values() metadata_files_count = sum(
if file_info.get("database_id") == db_id) 1 for file_info in chroma_kb.files_meta.values() if file_info.get("database_id") == db_id
)
# 如果向量数据库中有数据但 metadata 中没有文件记录,可能存在文件缺失 # 如果向量数据库中有数据但 metadata 中没有文件记录,可能存在文件缺失
if actual_count > 0 and metadata_files_count == 0: if actual_count > 0 and metadata_files_count == 0:
inconsistencies["missing_files"].append({ inconsistencies["missing_files"].append(
"database_id": db_id, {
"vector_count": actual_count, "database_id": db_id,
"metadata_files_count": metadata_files_count, "vector_count": actual_count,
"detected_at": utc_isoformat() "metadata_files_count": metadata_files_count,
}) "detected_at": utc_isoformat(),
logger.warning(f"发现数据库 {db_id} 在 ChromaDB 中有 {actual_count} 条向量数据,但 metadata 中没有文件记录") }
)
logger.warning(
f"发现数据库 {db_id} 在 ChromaDB 中有 {actual_count} 条向量数据,但 metadata 中没有文件记录"
)
except Exception as e: except Exception as e:
logger.debug(f"检查数据库 {db_id} 的文件一致性时出错: {e}") logger.debug(f"检查数据库 {db_id} 的文件一致性时出错: {e}")
@ -640,14 +640,12 @@ class KnowledgeBaseManager:
if not collection_name.startswith("kb_"): if not collection_name.startswith("kb_"):
continue continue
collection_info = { collection_info = {"collection_name": collection_name, "detected_at": utc_isoformat()}
"collection_name": collection_name,
"detected_at": utc_isoformat()
}
# 尝试获取集合的基本信息 # 尝试获取集合的基本信息
try: try:
from pymilvus import Collection from pymilvus import Collection
collection = Collection(name=collection_name, using=milvus_kb.connection_alias) collection = Collection(name=collection_name, using=milvus_kb.connection_alias)
collection_info["count"] = collection.num_entities collection_info["count"] = collection.num_entities
collection_info["description"] = collection.description collection_info["description"] = collection.description
@ -656,30 +654,39 @@ class KnowledgeBaseManager:
collection_info["count"] = "unknown" collection_info["count"] = "unknown"
inconsistencies["missing_collections"].append(collection_info) inconsistencies["missing_collections"].append(collection_info)
logger.warning(f"发现 Milvus 中存在但 metadata 中缺失的集合: {collection_name} (实体数: {collection_info['count']})") logger.warning(
f"发现 Milvus 中存在但 metadata 中缺失的集合: {collection_name} "
f"(实体数: {collection_info['count']})"
)
# 检查文件级别的不一致(针对已知的数据库) # 检查文件级别的不一致(针对已知的数据库)
for db_id in metadata_collection_names: for db_id in metadata_collection_names:
try: try:
if utility.has_collection(db_id, using=milvus_kb.connection_alias): if utility.has_collection(db_id, using=milvus_kb.connection_alias):
from pymilvus import Collection from pymilvus import Collection
collection = Collection(name=db_id, using=milvus_kb.connection_alias) collection = Collection(name=db_id, using=milvus_kb.connection_alias)
actual_count = collection.num_entities actual_count = collection.num_entities
# 获取 metadata 中记录的文件数量 # 获取 metadata 中记录的文件数量
metadata_files_count = sum(1 for file_info in milvus_kb.files_meta.values() metadata_files_count = sum(
if file_info.get("database_id") == db_id) 1 for file_info in milvus_kb.files_meta.values() if file_info.get("database_id") == db_id
)
# 如果向量数据库中有数据但 metadata 中没有文件记录,可能存在文件缺失 # 如果向量数据库中有数据但 metadata 中没有文件记录,可能存在文件缺失
if actual_count > 0 and metadata_files_count == 0: if actual_count > 0 and metadata_files_count == 0:
inconsistencies["missing_files"].append({ inconsistencies["missing_files"].append(
"database_id": db_id, {
"vector_count": actual_count, "database_id": db_id,
"metadata_files_count": metadata_files_count, "vector_count": actual_count,
"detected_at": utc_isoformat() "metadata_files_count": metadata_files_count,
}) "detected_at": utc_isoformat(),
logger.warning(f"发现数据库 {db_id} 在 Milvus 中有 {actual_count} 条向量数据,但 metadata 中没有文件记录") }
)
logger.warning(
f"发现数据库 {db_id} 在 Milvus 中有 {actual_count} 条向量数据,"
"但 metadata 中没有文件记录"
)
except Exception as e: except Exception as e:
logger.debug(f"检查数据库 {db_id} 的文件一致性时出错: {e}") logger.debug(f"检查数据库 {db_id} 的文件一致性时出错: {e}")
@ -706,25 +713,31 @@ class KnowledgeBaseManager:
chroma_missing = inconsistencies["chroma"]["missing_collections"] chroma_missing = inconsistencies["chroma"]["missing_collections"]
chroma_files_missing = inconsistencies["chroma"]["missing_files"] chroma_files_missing = inconsistencies["chroma"]["missing_files"]
if chroma_missing or chroma_files_missing: if chroma_missing or chroma_files_missing:
logger.warning(f"ChromaDB 不一致情况:") logger.warning("ChromaDB 不一致情况:")
logger.warning(f" 缺失集合数量: {len(chroma_missing)}") logger.warning(f" 缺失集合数量: {len(chroma_missing)}")
for collection_info in chroma_missing: for collection_info in chroma_missing:
logger.warning(f" - 集合: {collection_info['collection_name']}, 向量数: {collection_info['count']}") logger.warning(f" - 集合: {collection_info['collection_name']}, 向量数: {collection_info['count']}")
logger.warning(f" 缺失文件记录数量: {len(chroma_files_missing)}") logger.warning(f" 缺失文件记录数量: {len(chroma_files_missing)}")
for file_info in chroma_files_missing: for file_info in chroma_files_missing:
logger.warning(f" - 数据库: {file_info['database_id']}, 向量数: {file_info['vector_count']}, 元数据文件数: {file_info['metadata_files_count']}") logger.warning(
f" - 数据库: {file_info['database_id']}, 向量数: {file_info['vector_count']}, "
f"元数据文件数: {file_info['metadata_files_count']}"
)
# Milvus 不一致情况 # Milvus 不一致情况
milvus_missing = inconsistencies["milvus"]["missing_collections"] milvus_missing = inconsistencies["milvus"]["missing_collections"]
milvus_files_missing = inconsistencies["milvus"]["missing_files"] milvus_files_missing = inconsistencies["milvus"]["missing_files"]
if milvus_missing or milvus_files_missing: if milvus_missing or milvus_files_missing:
logger.warning(f"Milvus 不一致情况:") logger.warning("Milvus 不一致情况:")
logger.warning(f" 缺失集合数量: {len(milvus_missing)}") logger.warning(f" 缺失集合数量: {len(milvus_missing)}")
for collection_info in milvus_missing: for collection_info in milvus_missing:
logger.warning(f" - 集合: {collection_info['collection_name']}, 实体数: {collection_info['count']}") logger.warning(f" - 集合: {collection_info['collection_name']}, 实体数: {collection_info['count']}")
logger.warning(f" 缺失文件记录数量: {len(milvus_files_missing)}") logger.warning(f" 缺失文件记录数量: {len(milvus_files_missing)}")
for file_info in milvus_files_missing: for file_info in milvus_files_missing:
logger.warning(f" - 数据库: {file_info['database_id']}, 向量数: {file_info['vector_count']}, 元数据文件数: {file_info['metadata_files_count']}") logger.warning(
f" - 数据库: {file_info['database_id']}, 向量数: {file_info['vector_count']}, "
f"元数据文件数: {file_info['metadata_files_count']}"
)
logger.warning("=" * 80) logger.warning("=" * 80)
logger.warning(f"总计:缺失集合 {total_missing_collections} 个,缺失文件记录 {total_missing_files}") logger.warning(f"总计:缺失集合 {total_missing_collections} 个,缺失文件记录 {total_missing_files}")

View File

@ -211,7 +211,7 @@ def get_embedding_config(embed_info: dict) -> dict:
try: try:
if embed_info: if embed_info:
# 处理 embed_info 可能是字典或 EmbedModelInfo 对象的情况 # 处理 embed_info 可能是字典或 EmbedModelInfo 对象的情况
if hasattr(embed_info, 'name'): if hasattr(embed_info, "name"):
# EmbedModelInfo 对象 # EmbedModelInfo 对象
config_dict["model"] = embed_info.name config_dict["model"] = embed_info.name
config_dict["api_key"] = os.getenv(embed_info.api_key, embed_info.api_key) config_dict["api_key"] = os.getenv(embed_info.api_key, embed_info.api_key)

View File

@ -12,7 +12,8 @@ from src.storage.db.models import Conversation, ConversationStats, Message, Tool
from src.utils import logger from src.utils import logger
from src.utils.datetime_utils import utc_now from src.utils.datetime_utils import utc_now
#TODO:[未完成]待修改为异步版本 # TODO:[未完成]待修改为异步版本
class ConversationManager: class ConversationManager:
"""Manager for conversation storage operations""" """Manager for conversation storage operations"""

View File

@ -22,6 +22,7 @@ except ImportError:
# TODO:[优化建议]需要将数据库修改为异步的aiosqlite或者异步mysql缓存使用Redis存储 # TODO:[优化建议]需要将数据库修改为异步的aiosqlite或者异步mysql缓存使用Redis存储
# TODO:[已完成]为DBManager添加单例模式 # TODO:[已完成]为DBManager添加单例模式
class DBManager(metaclass=SingletonMeta): class DBManager(metaclass=SingletonMeta):
"""数据库管理器 - 只提供基础的数据库连接和会话管理""" """数据库管理器 - 只提供基础的数据库连接和会话管理"""