style: ruff format
This commit is contained in:
parent
92a110d955
commit
a1fdcbc36c
1
Makefile
1
Makefile
@ -27,6 +27,7 @@ lint:
|
||||
format:
|
||||
uv run ruff format .
|
||||
uv run ruff check . --fix
|
||||
uv run python -m ruff check --select I src --fix
|
||||
|
||||
format_diff:
|
||||
uv run ruff format --diff .
|
||||
|
||||
@ -112,7 +112,8 @@ async def get_agent(current_user: User = Depends(get_required_user)):
|
||||
metadata = yaml.safe_load(f)
|
||||
return {"agents": agents, "metadata": metadata}
|
||||
|
||||
#TODO:[未完成]这个thread_id在前端是直接生成的1234,最好传入thread_id时做校验只允许uuid4
|
||||
|
||||
# TODO:[未完成]这个thread_id在前端是直接生成的1234,最好传入thread_id时做校验只允许uuid4
|
||||
@chat.post("/agent/{agent_id}")
|
||||
async def chat_agent(
|
||||
agent_id: str,
|
||||
@ -254,7 +255,8 @@ async def chat_agent(
|
||||
logger.error(f"Error saving messages from LangGraph state: {e}")
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
#TODO:[功能建议]针对需要人工审批后再执行的工具,可以使用langgraph的interrupt方法中断对话,等待用户输入后再使用command跳转回去
|
||||
# TODO:[功能建议]针对需要人工审批后再执行的工具,
|
||||
# 可以使用langgraph的interrupt方法中断对话,等待用户输入后再使用command跳转回去
|
||||
async def stream_messages():
|
||||
# 代表服务端已经收到了请求
|
||||
yield make_chunk(status="init", meta=meta, msg=HumanMessage(content=query).model_dump())
|
||||
|
||||
@ -1,11 +1,11 @@
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from server.services import tasker
|
||||
|
||||
#TODO:[已完成]使用lifespan进行统一生命周期管理
|
||||
# TODO:[已完成]使用lifespan进行统一生命周期管理
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
from threading import Lock
|
||||
|
||||
|
||||
class SingletonMeta(type):
|
||||
"""
|
||||
This is a thread-safe implementation of Singleton.
|
||||
"""
|
||||
|
||||
_instances = {}
|
||||
_lock: Lock = Lock()
|
||||
|
||||
@ -12,4 +14,4 @@ class SingletonMeta(type):
|
||||
if cls not in cls._instances:
|
||||
instance = super().__call__(*args, **kwargs)
|
||||
cls._instances[cls] = instance
|
||||
return cls._instances[cls]
|
||||
return cls._instances[cls]
|
||||
|
||||
@ -9,8 +9,9 @@ from src.agents.common.tools import get_buildin_tools
|
||||
from src.storage.minio import upload_image_to_minio
|
||||
from src.utils import logger
|
||||
|
||||
#TODO:[已完成]修改了tool定义的示例,使用更符合langgraph调用的方式
|
||||
@tool(name_or_callable="全能计算器",description="可以对给定的2个数字选择进行加减乘除四种计算")
|
||||
|
||||
# TODO:[已完成]修改了tool定义的示例,使用更符合langgraph调用的方式
|
||||
@tool(name_or_callable="全能计算器", description="可以对给定的2个数字选择进行加减乘除四种计算")
|
||||
def calculator(a: float, b: float, operation: str) -> float:
|
||||
"""
|
||||
可以对给定的2个数字选择进行加减乘除四种计算
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from abc import abstractmethod
|
||||
from pathlib import Path
|
||||
|
||||
from langgraph.graph.state import CompiledStateGraph
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver, aiosqlite
|
||||
from langgraph.graph.state import CompiledStateGraph
|
||||
|
||||
from src import config as sys_config
|
||||
from src.agents.common.context import BaseContext
|
||||
|
||||
@ -11,7 +11,7 @@ from src import config, graph_base, knowledge_base
|
||||
from src.utils import logger
|
||||
|
||||
|
||||
@tool
|
||||
@tool(name_or_callable="查询知识图谱", description="使用这个工具可以查询知识图谱中包含的三元组信息。")
|
||||
def query_knowledge_graph(query: Annotated[str, "The keyword to query knowledge graph."]) -> Any:
|
||||
"""Use this to query knowledge graph, which include some food domain knowledge."""
|
||||
try:
|
||||
@ -35,7 +35,9 @@ def get_static_tools() -> list:
|
||||
|
||||
# 检查是否启用网页搜索
|
||||
if config.enable_web_search:
|
||||
static_tools.append(TavilySearch(max_results=10))
|
||||
search = TavilySearch(max_results=10)
|
||||
search.metadata = {"name": "Tavily 网页搜索"}
|
||||
static_tools.append(search)
|
||||
|
||||
return static_tools
|
||||
|
||||
@ -76,9 +78,6 @@ def get_kb_based_tools() -> list:
|
||||
|
||||
for db_id, retrieve_info in retrievers.items():
|
||||
try:
|
||||
# 使用改进的工具ID生成策略
|
||||
tool_id = f"query_{db_id[:8]}"
|
||||
|
||||
# 构建工具描述
|
||||
description = (
|
||||
f"使用 {retrieve_info['name']} 知识库进行检索。\n"
|
||||
@ -88,10 +87,12 @@ def get_kb_based_tools() -> list:
|
||||
# 使用工厂函数创建检索器包装函数,避免闭包问题
|
||||
retriever_wrapper = _create_retriever_wrapper(db_id, retrieve_info)
|
||||
|
||||
safename = retrieve_info["name"].replace(" ", "_")[:20]
|
||||
|
||||
# 使用 StructuredTool.from_function 创建异步工具
|
||||
tool = StructuredTool.from_function(
|
||||
coroutine=retriever_wrapper,
|
||||
name=tool_id,
|
||||
name=safename,
|
||||
description=description,
|
||||
args_schema=KnowledgeRetrieverModel,
|
||||
metadata=retrieve_info["metadata"] | {"tag": ["knowledgebase"]},
|
||||
|
||||
@ -13,7 +13,7 @@ from typing import Any
|
||||
|
||||
import tomli
|
||||
import tomli_w
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from src.config.static.models import (
|
||||
DEFAULT_CHAT_MODEL_PROVIDERS,
|
||||
@ -172,13 +172,10 @@ class Config(BaseModel):
|
||||
self.model_dir = os.environ.get("MODEL_DIR", self.model_dir)
|
||||
if self.model_dir:
|
||||
if os.path.exists(self.model_dir):
|
||||
logger.debug(
|
||||
f"Model directory ({self.model_dir}) contains: {os.listdir(self.model_dir)}"
|
||||
)
|
||||
logger.debug(f"Model directory ({self.model_dir}) contains: {os.listdir(self.model_dir)}")
|
||||
else:
|
||||
logger.warning(
|
||||
f"Model directory ({self.model_dir}) does not exist. "
|
||||
"If not configured, please ignore it."
|
||||
f"Model directory ({self.model_dir}) does not exist. If not configured, please ignore it."
|
||||
)
|
||||
|
||||
# 检查模型提供商的环境变量
|
||||
@ -195,14 +192,10 @@ class Config(BaseModel):
|
||||
self.enable_web_search = True
|
||||
|
||||
# 获取可用的模型提供商
|
||||
self.valuable_model_provider = [
|
||||
k for k, v in self.model_provider_status.items() if v
|
||||
]
|
||||
self.valuable_model_provider = [k for k, v in self.model_provider_status.items() if v]
|
||||
|
||||
if not self.valuable_model_provider:
|
||||
raise ValueError(
|
||||
"No model provider available, please check your `.env` file."
|
||||
)
|
||||
raise ValueError("No model provider available, please check your `.env` file.")
|
||||
|
||||
def save(self):
|
||||
"""保存配置到 TOML 文件(仅保存用户修改的字段)"""
|
||||
@ -251,15 +244,11 @@ class Config(BaseModel):
|
||||
)
|
||||
|
||||
# 添加模型信息(转换为字典格式供前端使用)
|
||||
config_dict["model_names"] = {
|
||||
provider: info.model_dump() for provider, info in self.model_names.items()
|
||||
}
|
||||
config_dict["model_names"] = {provider: info.model_dump() for provider, info in self.model_names.items()}
|
||||
config_dict["embed_model_names"] = {
|
||||
model_id: info.model_dump() for model_id, info in self.embed_model_names.items()
|
||||
}
|
||||
config_dict["reranker_names"] = {
|
||||
model_id: info.model_dump() for model_id, info in self.reranker_names.items()
|
||||
}
|
||||
config_dict["reranker_names"] = {model_id: info.model_dump() for model_id, info in self.reranker_names.items()}
|
||||
|
||||
# 添加运行时状态信息
|
||||
config_dict["model_provider_status"] = self.model_provider_status
|
||||
@ -269,10 +258,12 @@ class Config(BaseModel):
|
||||
for field_name, field_info in Config.model_fields.items():
|
||||
if not field_info.exclude: # 排除内部字段
|
||||
fields_info[field_name] = {
|
||||
'des': field_info.description,
|
||||
'default': field_info.default,
|
||||
'type': field_info.annotation.__name__ if hasattr(field_info.annotation, '__name__') else str(field_info.annotation),
|
||||
'exclude': field_info.exclude if hasattr(field_info, 'exclude') else False,
|
||||
"des": field_info.description,
|
||||
"default": field_info.default,
|
||||
"type": field_info.annotation.__name__
|
||||
if hasattr(field_info.annotation, "__name__")
|
||||
else str(field_info.annotation),
|
||||
"exclude": field_info.exclude if hasattr(field_info, "exclude") else False,
|
||||
}
|
||||
config_dict["_config_items"] = fields_info
|
||||
|
||||
@ -301,14 +292,12 @@ class Config(BaseModel):
|
||||
|
||||
def __getitem__(self, key: str) -> Any:
|
||||
"""支持字典式访问 config[key]"""
|
||||
logger.warning("Using deprecated dict-style access for Config. "
|
||||
"Please use attribute access instead.")
|
||||
logger.warning("Using deprecated dict-style access for Config. Please use attribute access instead.")
|
||||
return getattr(self, key, None)
|
||||
|
||||
def __setitem__(self, key: str, value: Any):
|
||||
"""支持字典式赋值 config[key] = value"""
|
||||
logger.warning("Using deprecated dict-style assignment for Config. "
|
||||
"Please use attribute access instead.")
|
||||
logger.warning("Using deprecated dict-style assignment for Config. Please use attribute access instead.")
|
||||
setattr(self, key, value)
|
||||
|
||||
def update(self, other: dict):
|
||||
@ -352,8 +341,7 @@ class Config(BaseModel):
|
||||
else:
|
||||
# 保存所有 model_names
|
||||
user_config["model_names"] = {
|
||||
provider: info.model_dump()
|
||||
for provider, info in self.model_names.items()
|
||||
provider: info.model_dump() for provider, info in self.model_names.items()
|
||||
}
|
||||
# 记录整个 model_names 字段的修改
|
||||
self._user_modified_fields.add("model_names")
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import shutil
|
||||
import tempfile
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
@ -144,8 +144,8 @@ class KnowledgeBase(ABC):
|
||||
"name": database_name,
|
||||
"description": description,
|
||||
"kb_type": self.kb_type,
|
||||
"embed_info": embed_info.model_dump() if hasattr(embed_info, 'model_dump') else embed_info,
|
||||
"llm_info": llm_info.model_dump() if hasattr(llm_info, 'model_dump') else llm_info,
|
||||
"embed_info": embed_info.model_dump() if hasattr(embed_info, "model_dump") else embed_info,
|
||||
"llm_info": llm_info.model_dump() if hasattr(llm_info, "model_dump") else llm_info,
|
||||
"metadata": kwargs,
|
||||
"created_at": utc_isoformat(),
|
||||
}
|
||||
@ -559,7 +559,7 @@ class KnowledgeBase(ABC):
|
||||
|
||||
def _serialize_metadata(self, obj):
|
||||
"""递归序列化元数据中的 Pydantic 模型"""
|
||||
if hasattr(obj, 'dict'):
|
||||
if hasattr(obj, "dict"):
|
||||
return obj.dict()
|
||||
elif isinstance(obj, dict):
|
||||
return {k: self._serialize_metadata(v) for k, v in obj.items()}
|
||||
@ -589,8 +589,7 @@ class KnowledgeBase(ABC):
|
||||
|
||||
# 原子性写入(使用临时文件)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode='w', dir=os.path.dirname(meta_file),
|
||||
prefix='.tmp_', suffix='.json', delete=False
|
||||
mode="w", dir=os.path.dirname(meta_file), prefix=".tmp_", suffix=".json", delete=False
|
||||
) as tmp_file:
|
||||
json.dump(data, tmp_file, ensure_ascii=False, indent=2)
|
||||
temp_path = tmp_file.name
|
||||
|
||||
@ -299,7 +299,7 @@ class GraphDatabase:
|
||||
logger.info(f"Adding entity to {kgdb_name}")
|
||||
session.execute_write(_create_graph, triples)
|
||||
logger.info(f"Creating vector index for {kgdb_name} with {config.embed_model}")
|
||||
session.execute_write(_create_vector_index, getattr(cur_embed_info, 'dimension', 1024))
|
||||
session.execute_write(_create_vector_index, getattr(cur_embed_info, "dimension", 1024))
|
||||
|
||||
# 收集所有需要处理的实体名称,去重
|
||||
all_entities = []
|
||||
|
||||
@ -72,11 +72,11 @@ class ChromaKB(KnowledgeBase):
|
||||
logger.info(f"Retrieved existing collection: {collection_name}")
|
||||
|
||||
# 检查现有集合的配置是否匹配当前的 embed_info
|
||||
expected_model = getattr(embed_info, 'name', None) if embed_info else None
|
||||
if expected_model is None and hasattr(embed_info, 'get'):
|
||||
expected_model = embed_info.get('name')
|
||||
expected_model = getattr(embed_info, "name", None) if embed_info else None
|
||||
if expected_model is None and hasattr(embed_info, "get"):
|
||||
expected_model = embed_info.get("name")
|
||||
elif embed_info and isinstance(embed_info, dict):
|
||||
expected_model = embed_info.get('name')
|
||||
expected_model = embed_info.get("name")
|
||||
expected_model = expected_model or "default"
|
||||
collection_metadata = collection.metadata or {}
|
||||
current_model = collection_metadata.get("embedding_model", "unknown")
|
||||
@ -93,13 +93,13 @@ class ChromaKB(KnowledgeBase):
|
||||
|
||||
except Exception:
|
||||
# 创建新集合
|
||||
model_name = getattr(embed_info, 'name', None) if embed_info else None
|
||||
if model_name is None and hasattr(embed_info, 'get'):
|
||||
model_name = embed_info.get('name')
|
||||
model_name = getattr(embed_info, "name", None) if embed_info else None
|
||||
if model_name is None and hasattr(embed_info, "get"):
|
||||
model_name = embed_info.get("name")
|
||||
elif embed_info and isinstance(embed_info, dict):
|
||||
model_name = embed_info.get('name')
|
||||
model_name = embed_info.get("name")
|
||||
|
||||
model_name = model_name or 'default'
|
||||
model_name = model_name or "default"
|
||||
logger.info(f"Creating new collection with embedding model: {model_name}")
|
||||
collection_metadata = {
|
||||
"db_id": db_id,
|
||||
|
||||
@ -103,7 +103,7 @@ class MilvusKB(KnowledgeBase):
|
||||
|
||||
# 检查嵌入模型是否匹配
|
||||
description = collection.description
|
||||
expected_model = getattr(embed_info, 'name', 'default') if embed_info else "default"
|
||||
expected_model = getattr(embed_info, "name", "default") if embed_info else "default"
|
||||
|
||||
if expected_model not in description:
|
||||
logger.warning(f"Collection {collection_name} model mismatch, recreating...")
|
||||
@ -116,8 +116,8 @@ class MilvusKB(KnowledgeBase):
|
||||
|
||||
except Exception:
|
||||
# 创建新集合
|
||||
embedding_dim = getattr(embed_info, 'dimension', 1024) if embed_info else 1024
|
||||
model_name = getattr(embed_info, 'name', 'default') if embed_info else "default"
|
||||
embedding_dim = getattr(embed_info, "dimension", 1024) if embed_info else 1024
|
||||
model_name = getattr(embed_info, "name", "default") if embed_info else "default"
|
||||
|
||||
# 定义集合Schema
|
||||
fields = [
|
||||
|
||||
@ -2,7 +2,6 @@ import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
||||
from langchain_community.document_loaders import (
|
||||
CSVLoader,
|
||||
JSONLoader,
|
||||
@ -12,10 +11,10 @@ from langchain_community.document_loaders import (
|
||||
UnstructuredMarkdownLoader,
|
||||
UnstructuredWordDocumentLoader,
|
||||
)
|
||||
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
||||
|
||||
from src.utils import logger
|
||||
|
||||
|
||||
SUPPORTED_FILE_EXTENSIONS: tuple[str, ...] = (
|
||||
".txt",
|
||||
".md",
|
||||
|
||||
@ -104,16 +104,11 @@ class KnowledgeBaseManager:
|
||||
shutil.copy2(meta_file, backup_file)
|
||||
|
||||
# 准备数据
|
||||
data = {
|
||||
"databases": self.global_databases_meta,
|
||||
"updated_at": utc_isoformat(),
|
||||
"version": "2.0"
|
||||
}
|
||||
data = {"databases": self.global_databases_meta, "updated_at": utc_isoformat(), "version": "2.0"}
|
||||
|
||||
# 原子性写入(使用临时文件)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode='w', dir=os.path.dirname(meta_file),
|
||||
prefix='.tmp_', suffix='.json', delete=False
|
||||
mode="w", dir=os.path.dirname(meta_file), prefix=".tmp_", suffix=".json", delete=False
|
||||
) as tmp_file:
|
||||
json.dump(data, tmp_file, ensure_ascii=False, indent=2)
|
||||
temp_path = tmp_file.name
|
||||
@ -520,7 +515,7 @@ class KnowledgeBaseManager:
|
||||
"chroma": {"missing_collections": [], "missing_files": []},
|
||||
"milvus": {"missing_collections": [], "missing_files": []},
|
||||
"total_missing_collections": 0,
|
||||
"total_missing_files": 0
|
||||
"total_missing_files": 0,
|
||||
}
|
||||
|
||||
logger.info("开始检测向量数据库与元数据的一致性...")
|
||||
@ -573,10 +568,7 @@ class KnowledgeBaseManager:
|
||||
if not collection_name.startswith("kb_"):
|
||||
continue
|
||||
|
||||
collection_info = {
|
||||
"collection_name": collection_name,
|
||||
"detected_at": utc_isoformat()
|
||||
}
|
||||
collection_info = {"collection_name": collection_name, "detected_at": utc_isoformat()}
|
||||
|
||||
# 尝试获取集合的基本信息
|
||||
try:
|
||||
@ -588,7 +580,10 @@ class KnowledgeBaseManager:
|
||||
collection_info["count"] = "unknown"
|
||||
|
||||
inconsistencies["missing_collections"].append(collection_info)
|
||||
logger.warning(f"发现 ChromaDB 中存在但 metadata 中缺失的集合: {collection_name} (文档数: {collection_info['count']})")
|
||||
logger.warning(
|
||||
f"发现 ChromaDB 中存在但 metadata 中缺失的集合: {collection_name} "
|
||||
f"(文档数: {collection_info['count']})"
|
||||
)
|
||||
|
||||
# 检查文件级别的不一致(针对已知的数据库)
|
||||
for db_id in metadata_collection_names:
|
||||
@ -597,18 +592,23 @@ class KnowledgeBaseManager:
|
||||
actual_count = collection.count()
|
||||
|
||||
# 获取 metadata 中记录的文件数量
|
||||
metadata_files_count = sum(1 for file_info in chroma_kb.files_meta.values()
|
||||
if file_info.get("database_id") == db_id)
|
||||
metadata_files_count = sum(
|
||||
1 for file_info in chroma_kb.files_meta.values() if file_info.get("database_id") == db_id
|
||||
)
|
||||
|
||||
# 如果向量数据库中有数据但 metadata 中没有文件记录,可能存在文件缺失
|
||||
if actual_count > 0 and metadata_files_count == 0:
|
||||
inconsistencies["missing_files"].append({
|
||||
"database_id": db_id,
|
||||
"vector_count": actual_count,
|
||||
"metadata_files_count": metadata_files_count,
|
||||
"detected_at": utc_isoformat()
|
||||
})
|
||||
logger.warning(f"发现数据库 {db_id} 在 ChromaDB 中有 {actual_count} 条向量数据,但 metadata 中没有文件记录")
|
||||
inconsistencies["missing_files"].append(
|
||||
{
|
||||
"database_id": db_id,
|
||||
"vector_count": actual_count,
|
||||
"metadata_files_count": metadata_files_count,
|
||||
"detected_at": utc_isoformat(),
|
||||
}
|
||||
)
|
||||
logger.warning(
|
||||
f"发现数据库 {db_id} 在 ChromaDB 中有 {actual_count} 条向量数据,但 metadata 中没有文件记录"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"检查数据库 {db_id} 的文件一致性时出错: {e}")
|
||||
@ -640,14 +640,12 @@ class KnowledgeBaseManager:
|
||||
if not collection_name.startswith("kb_"):
|
||||
continue
|
||||
|
||||
collection_info = {
|
||||
"collection_name": collection_name,
|
||||
"detected_at": utc_isoformat()
|
||||
}
|
||||
collection_info = {"collection_name": collection_name, "detected_at": utc_isoformat()}
|
||||
|
||||
# 尝试获取集合的基本信息
|
||||
try:
|
||||
from pymilvus import Collection
|
||||
|
||||
collection = Collection(name=collection_name, using=milvus_kb.connection_alias)
|
||||
collection_info["count"] = collection.num_entities
|
||||
collection_info["description"] = collection.description
|
||||
@ -656,30 +654,39 @@ class KnowledgeBaseManager:
|
||||
collection_info["count"] = "unknown"
|
||||
|
||||
inconsistencies["missing_collections"].append(collection_info)
|
||||
logger.warning(f"发现 Milvus 中存在但 metadata 中缺失的集合: {collection_name} (实体数: {collection_info['count']})")
|
||||
|
||||
logger.warning(
|
||||
f"发现 Milvus 中存在但 metadata 中缺失的集合: {collection_name} "
|
||||
f"(实体数: {collection_info['count']})"
|
||||
)
|
||||
|
||||
# 检查文件级别的不一致(针对已知的数据库)
|
||||
for db_id in metadata_collection_names:
|
||||
try:
|
||||
if utility.has_collection(db_id, using=milvus_kb.connection_alias):
|
||||
from pymilvus import Collection
|
||||
|
||||
collection = Collection(name=db_id, using=milvus_kb.connection_alias)
|
||||
actual_count = collection.num_entities
|
||||
|
||||
# 获取 metadata 中记录的文件数量
|
||||
metadata_files_count = sum(1 for file_info in milvus_kb.files_meta.values()
|
||||
if file_info.get("database_id") == db_id)
|
||||
metadata_files_count = sum(
|
||||
1 for file_info in milvus_kb.files_meta.values() if file_info.get("database_id") == db_id
|
||||
)
|
||||
|
||||
# 如果向量数据库中有数据但 metadata 中没有文件记录,可能存在文件缺失
|
||||
if actual_count > 0 and metadata_files_count == 0:
|
||||
inconsistencies["missing_files"].append({
|
||||
"database_id": db_id,
|
||||
"vector_count": actual_count,
|
||||
"metadata_files_count": metadata_files_count,
|
||||
"detected_at": utc_isoformat()
|
||||
})
|
||||
logger.warning(f"发现数据库 {db_id} 在 Milvus 中有 {actual_count} 条向量数据,但 metadata 中没有文件记录")
|
||||
inconsistencies["missing_files"].append(
|
||||
{
|
||||
"database_id": db_id,
|
||||
"vector_count": actual_count,
|
||||
"metadata_files_count": metadata_files_count,
|
||||
"detected_at": utc_isoformat(),
|
||||
}
|
||||
)
|
||||
logger.warning(
|
||||
f"发现数据库 {db_id} 在 Milvus 中有 {actual_count} 条向量数据,"
|
||||
"但 metadata 中没有文件记录"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"检查数据库 {db_id} 的文件一致性时出错: {e}")
|
||||
@ -706,25 +713,31 @@ class KnowledgeBaseManager:
|
||||
chroma_missing = inconsistencies["chroma"]["missing_collections"]
|
||||
chroma_files_missing = inconsistencies["chroma"]["missing_files"]
|
||||
if chroma_missing or chroma_files_missing:
|
||||
logger.warning(f"ChromaDB 不一致情况:")
|
||||
logger.warning("ChromaDB 不一致情况:")
|
||||
logger.warning(f" 缺失集合数量: {len(chroma_missing)}")
|
||||
for collection_info in chroma_missing:
|
||||
logger.warning(f" - 集合: {collection_info['collection_name']}, 向量数: {collection_info['count']}")
|
||||
logger.warning(f" 缺失文件记录数量: {len(chroma_files_missing)}")
|
||||
for file_info in chroma_files_missing:
|
||||
logger.warning(f" - 数据库: {file_info['database_id']}, 向量数: {file_info['vector_count']}, 元数据文件数: {file_info['metadata_files_count']}")
|
||||
logger.warning(
|
||||
f" - 数据库: {file_info['database_id']}, 向量数: {file_info['vector_count']}, "
|
||||
f"元数据文件数: {file_info['metadata_files_count']}"
|
||||
)
|
||||
|
||||
# Milvus 不一致情况
|
||||
milvus_missing = inconsistencies["milvus"]["missing_collections"]
|
||||
milvus_files_missing = inconsistencies["milvus"]["missing_files"]
|
||||
if milvus_missing or milvus_files_missing:
|
||||
logger.warning(f"Milvus 不一致情况:")
|
||||
logger.warning("Milvus 不一致情况:")
|
||||
logger.warning(f" 缺失集合数量: {len(milvus_missing)}")
|
||||
for collection_info in milvus_missing:
|
||||
logger.warning(f" - 集合: {collection_info['collection_name']}, 实体数: {collection_info['count']}")
|
||||
logger.warning(f" 缺失文件记录数量: {len(milvus_files_missing)}")
|
||||
for file_info in milvus_files_missing:
|
||||
logger.warning(f" - 数据库: {file_info['database_id']}, 向量数: {file_info['vector_count']}, 元数据文件数: {file_info['metadata_files_count']}")
|
||||
logger.warning(
|
||||
f" - 数据库: {file_info['database_id']}, 向量数: {file_info['vector_count']}, "
|
||||
f"元数据文件数: {file_info['metadata_files_count']}"
|
||||
)
|
||||
|
||||
logger.warning("=" * 80)
|
||||
logger.warning(f"总计:缺失集合 {total_missing_collections} 个,缺失文件记录 {total_missing_files} 个")
|
||||
|
||||
@ -211,7 +211,7 @@ def get_embedding_config(embed_info: dict) -> dict:
|
||||
try:
|
||||
if embed_info:
|
||||
# 处理 embed_info 可能是字典或 EmbedModelInfo 对象的情况
|
||||
if hasattr(embed_info, 'name'):
|
||||
if hasattr(embed_info, "name"):
|
||||
# EmbedModelInfo 对象
|
||||
config_dict["model"] = embed_info.name
|
||||
config_dict["api_key"] = os.getenv(embed_info.api_key, embed_info.api_key)
|
||||
|
||||
@ -12,7 +12,8 @@ from src.storage.db.models import Conversation, ConversationStats, Message, Tool
|
||||
from src.utils import logger
|
||||
from src.utils.datetime_utils import utc_now
|
||||
|
||||
#TODO:[未完成]待修改为异步版本
|
||||
# TODO:[未完成]待修改为异步版本
|
||||
|
||||
|
||||
class ConversationManager:
|
||||
"""Manager for conversation storage operations"""
|
||||
|
||||
@ -22,6 +22,7 @@ except ImportError:
|
||||
# TODO:[优化建议]需要将数据库修改为异步的aiosqlite或者异步mysql,缓存使用Redis存储
|
||||
# TODO:[已完成]为DBManager添加单例模式
|
||||
|
||||
|
||||
class DBManager(metaclass=SingletonMeta):
|
||||
"""数据库管理器 - 只提供基础的数据库连接和会话管理"""
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user