style: ruff format
This commit is contained in:
parent
92a110d955
commit
a1fdcbc36c
1
Makefile
1
Makefile
@ -27,6 +27,7 @@ lint:
|
|||||||
format:
|
format:
|
||||||
uv run ruff format .
|
uv run ruff format .
|
||||||
uv run ruff check . --fix
|
uv run ruff check . --fix
|
||||||
|
uv run python -m ruff check --select I src --fix
|
||||||
|
|
||||||
format_diff:
|
format_diff:
|
||||||
uv run ruff format --diff .
|
uv run ruff format --diff .
|
||||||
|
|||||||
@ -112,7 +112,8 @@ async def get_agent(current_user: User = Depends(get_required_user)):
|
|||||||
metadata = yaml.safe_load(f)
|
metadata = yaml.safe_load(f)
|
||||||
return {"agents": agents, "metadata": metadata}
|
return {"agents": agents, "metadata": metadata}
|
||||||
|
|
||||||
#TODO:[未完成]这个thread_id在前端是直接生成的1234,最好传入thread_id时做校验只允许uuid4
|
|
||||||
|
# TODO:[未完成]这个thread_id在前端是直接生成的1234,最好传入thread_id时做校验只允许uuid4
|
||||||
@chat.post("/agent/{agent_id}")
|
@chat.post("/agent/{agent_id}")
|
||||||
async def chat_agent(
|
async def chat_agent(
|
||||||
agent_id: str,
|
agent_id: str,
|
||||||
@ -254,7 +255,8 @@ async def chat_agent(
|
|||||||
logger.error(f"Error saving messages from LangGraph state: {e}")
|
logger.error(f"Error saving messages from LangGraph state: {e}")
|
||||||
logger.error(traceback.format_exc())
|
logger.error(traceback.format_exc())
|
||||||
|
|
||||||
#TODO:[功能建议]针对需要人工审批后再执行的工具,可以使用langgraph的interrupt方法中断对话,等待用户输入后再使用command跳转回去
|
# TODO:[功能建议]针对需要人工审批后再执行的工具,
|
||||||
|
# 可以使用langgraph的interrupt方法中断对话,等待用户输入后再使用command跳转回去
|
||||||
async def stream_messages():
|
async def stream_messages():
|
||||||
# 代表服务端已经收到了请求
|
# 代表服务端已经收到了请求
|
||||||
yield make_chunk(status="init", meta=meta, msg=HumanMessage(content=query).model_dump())
|
yield make_chunk(status="init", meta=meta, msg=HumanMessage(content=query).model_dump())
|
||||||
|
|||||||
@ -1,11 +1,11 @@
|
|||||||
import asyncio
|
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
|
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
|
|
||||||
from server.services import tasker
|
from server.services import tasker
|
||||||
|
|
||||||
#TODO:[已完成]使用lifespan进行统一生命周期管理
|
# TODO:[已完成]使用lifespan进行统一生命周期管理
|
||||||
|
|
||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def lifespan(app: FastAPI):
|
async def lifespan(app: FastAPI):
|
||||||
|
|||||||
@ -1,9 +1,11 @@
|
|||||||
from threading import Lock
|
from threading import Lock
|
||||||
|
|
||||||
|
|
||||||
class SingletonMeta(type):
|
class SingletonMeta(type):
|
||||||
"""
|
"""
|
||||||
This is a thread-safe implementation of Singleton.
|
This is a thread-safe implementation of Singleton.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_instances = {}
|
_instances = {}
|
||||||
_lock: Lock = Lock()
|
_lock: Lock = Lock()
|
||||||
|
|
||||||
@ -12,4 +14,4 @@ class SingletonMeta(type):
|
|||||||
if cls not in cls._instances:
|
if cls not in cls._instances:
|
||||||
instance = super().__call__(*args, **kwargs)
|
instance = super().__call__(*args, **kwargs)
|
||||||
cls._instances[cls] = instance
|
cls._instances[cls] = instance
|
||||||
return cls._instances[cls]
|
return cls._instances[cls]
|
||||||
|
|||||||
@ -9,8 +9,9 @@ from src.agents.common.tools import get_buildin_tools
|
|||||||
from src.storage.minio import upload_image_to_minio
|
from src.storage.minio import upload_image_to_minio
|
||||||
from src.utils import logger
|
from src.utils import logger
|
||||||
|
|
||||||
#TODO:[已完成]修改了tool定义的示例,使用更符合langgraph调用的方式
|
|
||||||
@tool(name_or_callable="全能计算器",description="可以对给定的2个数字选择进行加减乘除四种计算")
|
# TODO:[已完成]修改了tool定义的示例,使用更符合langgraph调用的方式
|
||||||
|
@tool(name_or_callable="全能计算器", description="可以对给定的2个数字选择进行加减乘除四种计算")
|
||||||
def calculator(a: float, b: float, operation: str) -> float:
|
def calculator(a: float, b: float, operation: str) -> float:
|
||||||
"""
|
"""
|
||||||
可以对给定的2个数字选择进行加减乘除四种计算
|
可以对给定的2个数字选择进行加减乘除四种计算
|
||||||
|
|||||||
@ -1,12 +1,12 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
|
||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from langgraph.graph.state import CompiledStateGraph
|
|
||||||
from langgraph.checkpoint.memory import InMemorySaver
|
from langgraph.checkpoint.memory import InMemorySaver
|
||||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver, aiosqlite
|
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver, aiosqlite
|
||||||
|
from langgraph.graph.state import CompiledStateGraph
|
||||||
|
|
||||||
from src import config as sys_config
|
from src import config as sys_config
|
||||||
from src.agents.common.context import BaseContext
|
from src.agents.common.context import BaseContext
|
||||||
|
|||||||
@ -11,7 +11,7 @@ from src import config, graph_base, knowledge_base
|
|||||||
from src.utils import logger
|
from src.utils import logger
|
||||||
|
|
||||||
|
|
||||||
@tool
|
@tool(name_or_callable="查询知识图谱", description="使用这个工具可以查询知识图谱中包含的三元组信息。")
|
||||||
def query_knowledge_graph(query: Annotated[str, "The keyword to query knowledge graph."]) -> Any:
|
def query_knowledge_graph(query: Annotated[str, "The keyword to query knowledge graph."]) -> Any:
|
||||||
"""Use this to query knowledge graph, which include some food domain knowledge."""
|
"""Use this to query knowledge graph, which include some food domain knowledge."""
|
||||||
try:
|
try:
|
||||||
@ -35,7 +35,9 @@ def get_static_tools() -> list:
|
|||||||
|
|
||||||
# 检查是否启用网页搜索
|
# 检查是否启用网页搜索
|
||||||
if config.enable_web_search:
|
if config.enable_web_search:
|
||||||
static_tools.append(TavilySearch(max_results=10))
|
search = TavilySearch(max_results=10)
|
||||||
|
search.metadata = {"name": "Tavily 网页搜索"}
|
||||||
|
static_tools.append(search)
|
||||||
|
|
||||||
return static_tools
|
return static_tools
|
||||||
|
|
||||||
@ -76,9 +78,6 @@ def get_kb_based_tools() -> list:
|
|||||||
|
|
||||||
for db_id, retrieve_info in retrievers.items():
|
for db_id, retrieve_info in retrievers.items():
|
||||||
try:
|
try:
|
||||||
# 使用改进的工具ID生成策略
|
|
||||||
tool_id = f"query_{db_id[:8]}"
|
|
||||||
|
|
||||||
# 构建工具描述
|
# 构建工具描述
|
||||||
description = (
|
description = (
|
||||||
f"使用 {retrieve_info['name']} 知识库进行检索。\n"
|
f"使用 {retrieve_info['name']} 知识库进行检索。\n"
|
||||||
@ -88,10 +87,12 @@ def get_kb_based_tools() -> list:
|
|||||||
# 使用工厂函数创建检索器包装函数,避免闭包问题
|
# 使用工厂函数创建检索器包装函数,避免闭包问题
|
||||||
retriever_wrapper = _create_retriever_wrapper(db_id, retrieve_info)
|
retriever_wrapper = _create_retriever_wrapper(db_id, retrieve_info)
|
||||||
|
|
||||||
|
safename = retrieve_info["name"].replace(" ", "_")[:20]
|
||||||
|
|
||||||
# 使用 StructuredTool.from_function 创建异步工具
|
# 使用 StructuredTool.from_function 创建异步工具
|
||||||
tool = StructuredTool.from_function(
|
tool = StructuredTool.from_function(
|
||||||
coroutine=retriever_wrapper,
|
coroutine=retriever_wrapper,
|
||||||
name=tool_id,
|
name=safename,
|
||||||
description=description,
|
description=description,
|
||||||
args_schema=KnowledgeRetrieverModel,
|
args_schema=KnowledgeRetrieverModel,
|
||||||
metadata=retrieve_info["metadata"] | {"tag": ["knowledgebase"]},
|
metadata=retrieve_info["metadata"] | {"tag": ["knowledgebase"]},
|
||||||
|
|||||||
@ -13,7 +13,7 @@ from typing import Any
|
|||||||
|
|
||||||
import tomli
|
import tomli
|
||||||
import tomli_w
|
import tomli_w
|
||||||
from pydantic import BaseModel, Field, field_validator
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
from src.config.static.models import (
|
from src.config.static.models import (
|
||||||
DEFAULT_CHAT_MODEL_PROVIDERS,
|
DEFAULT_CHAT_MODEL_PROVIDERS,
|
||||||
@ -172,13 +172,10 @@ class Config(BaseModel):
|
|||||||
self.model_dir = os.environ.get("MODEL_DIR", self.model_dir)
|
self.model_dir = os.environ.get("MODEL_DIR", self.model_dir)
|
||||||
if self.model_dir:
|
if self.model_dir:
|
||||||
if os.path.exists(self.model_dir):
|
if os.path.exists(self.model_dir):
|
||||||
logger.debug(
|
logger.debug(f"Model directory ({self.model_dir}) contains: {os.listdir(self.model_dir)}")
|
||||||
f"Model directory ({self.model_dir}) contains: {os.listdir(self.model_dir)}"
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f"Model directory ({self.model_dir}) does not exist. "
|
f"Model directory ({self.model_dir}) does not exist. If not configured, please ignore it."
|
||||||
"If not configured, please ignore it."
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# 检查模型提供商的环境变量
|
# 检查模型提供商的环境变量
|
||||||
@ -195,14 +192,10 @@ class Config(BaseModel):
|
|||||||
self.enable_web_search = True
|
self.enable_web_search = True
|
||||||
|
|
||||||
# 获取可用的模型提供商
|
# 获取可用的模型提供商
|
||||||
self.valuable_model_provider = [
|
self.valuable_model_provider = [k for k, v in self.model_provider_status.items() if v]
|
||||||
k for k, v in self.model_provider_status.items() if v
|
|
||||||
]
|
|
||||||
|
|
||||||
if not self.valuable_model_provider:
|
if not self.valuable_model_provider:
|
||||||
raise ValueError(
|
raise ValueError("No model provider available, please check your `.env` file.")
|
||||||
"No model provider available, please check your `.env` file."
|
|
||||||
)
|
|
||||||
|
|
||||||
def save(self):
|
def save(self):
|
||||||
"""保存配置到 TOML 文件(仅保存用户修改的字段)"""
|
"""保存配置到 TOML 文件(仅保存用户修改的字段)"""
|
||||||
@ -251,15 +244,11 @@ class Config(BaseModel):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# 添加模型信息(转换为字典格式供前端使用)
|
# 添加模型信息(转换为字典格式供前端使用)
|
||||||
config_dict["model_names"] = {
|
config_dict["model_names"] = {provider: info.model_dump() for provider, info in self.model_names.items()}
|
||||||
provider: info.model_dump() for provider, info in self.model_names.items()
|
|
||||||
}
|
|
||||||
config_dict["embed_model_names"] = {
|
config_dict["embed_model_names"] = {
|
||||||
model_id: info.model_dump() for model_id, info in self.embed_model_names.items()
|
model_id: info.model_dump() for model_id, info in self.embed_model_names.items()
|
||||||
}
|
}
|
||||||
config_dict["reranker_names"] = {
|
config_dict["reranker_names"] = {model_id: info.model_dump() for model_id, info in self.reranker_names.items()}
|
||||||
model_id: info.model_dump() for model_id, info in self.reranker_names.items()
|
|
||||||
}
|
|
||||||
|
|
||||||
# 添加运行时状态信息
|
# 添加运行时状态信息
|
||||||
config_dict["model_provider_status"] = self.model_provider_status
|
config_dict["model_provider_status"] = self.model_provider_status
|
||||||
@ -269,10 +258,12 @@ class Config(BaseModel):
|
|||||||
for field_name, field_info in Config.model_fields.items():
|
for field_name, field_info in Config.model_fields.items():
|
||||||
if not field_info.exclude: # 排除内部字段
|
if not field_info.exclude: # 排除内部字段
|
||||||
fields_info[field_name] = {
|
fields_info[field_name] = {
|
||||||
'des': field_info.description,
|
"des": field_info.description,
|
||||||
'default': field_info.default,
|
"default": field_info.default,
|
||||||
'type': field_info.annotation.__name__ if hasattr(field_info.annotation, '__name__') else str(field_info.annotation),
|
"type": field_info.annotation.__name__
|
||||||
'exclude': field_info.exclude if hasattr(field_info, 'exclude') else False,
|
if hasattr(field_info.annotation, "__name__")
|
||||||
|
else str(field_info.annotation),
|
||||||
|
"exclude": field_info.exclude if hasattr(field_info, "exclude") else False,
|
||||||
}
|
}
|
||||||
config_dict["_config_items"] = fields_info
|
config_dict["_config_items"] = fields_info
|
||||||
|
|
||||||
@ -301,14 +292,12 @@ class Config(BaseModel):
|
|||||||
|
|
||||||
def __getitem__(self, key: str) -> Any:
|
def __getitem__(self, key: str) -> Any:
|
||||||
"""支持字典式访问 config[key]"""
|
"""支持字典式访问 config[key]"""
|
||||||
logger.warning("Using deprecated dict-style access for Config. "
|
logger.warning("Using deprecated dict-style access for Config. Please use attribute access instead.")
|
||||||
"Please use attribute access instead.")
|
|
||||||
return getattr(self, key, None)
|
return getattr(self, key, None)
|
||||||
|
|
||||||
def __setitem__(self, key: str, value: Any):
|
def __setitem__(self, key: str, value: Any):
|
||||||
"""支持字典式赋值 config[key] = value"""
|
"""支持字典式赋值 config[key] = value"""
|
||||||
logger.warning("Using deprecated dict-style assignment for Config. "
|
logger.warning("Using deprecated dict-style assignment for Config. Please use attribute access instead.")
|
||||||
"Please use attribute access instead.")
|
|
||||||
setattr(self, key, value)
|
setattr(self, key, value)
|
||||||
|
|
||||||
def update(self, other: dict):
|
def update(self, other: dict):
|
||||||
@ -352,8 +341,7 @@ class Config(BaseModel):
|
|||||||
else:
|
else:
|
||||||
# 保存所有 model_names
|
# 保存所有 model_names
|
||||||
user_config["model_names"] = {
|
user_config["model_names"] = {
|
||||||
provider: info.model_dump()
|
provider: info.model_dump() for provider, info in self.model_names.items()
|
||||||
for provider, info in self.model_names.items()
|
|
||||||
}
|
}
|
||||||
# 记录整个 model_names 字段的修改
|
# 记录整个 model_names 字段的修改
|
||||||
self._user_modified_fields.add("model_names")
|
self._user_modified_fields.add("model_names")
|
||||||
|
|||||||
@ -1,7 +1,7 @@
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import tempfile
|
|
||||||
import shutil
|
import shutil
|
||||||
|
import tempfile
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@ -144,8 +144,8 @@ class KnowledgeBase(ABC):
|
|||||||
"name": database_name,
|
"name": database_name,
|
||||||
"description": description,
|
"description": description,
|
||||||
"kb_type": self.kb_type,
|
"kb_type": self.kb_type,
|
||||||
"embed_info": embed_info.model_dump() if hasattr(embed_info, 'model_dump') else embed_info,
|
"embed_info": embed_info.model_dump() if hasattr(embed_info, "model_dump") else embed_info,
|
||||||
"llm_info": llm_info.model_dump() if hasattr(llm_info, 'model_dump') else llm_info,
|
"llm_info": llm_info.model_dump() if hasattr(llm_info, "model_dump") else llm_info,
|
||||||
"metadata": kwargs,
|
"metadata": kwargs,
|
||||||
"created_at": utc_isoformat(),
|
"created_at": utc_isoformat(),
|
||||||
}
|
}
|
||||||
@ -559,7 +559,7 @@ class KnowledgeBase(ABC):
|
|||||||
|
|
||||||
def _serialize_metadata(self, obj):
|
def _serialize_metadata(self, obj):
|
||||||
"""递归序列化元数据中的 Pydantic 模型"""
|
"""递归序列化元数据中的 Pydantic 模型"""
|
||||||
if hasattr(obj, 'dict'):
|
if hasattr(obj, "dict"):
|
||||||
return obj.dict()
|
return obj.dict()
|
||||||
elif isinstance(obj, dict):
|
elif isinstance(obj, dict):
|
||||||
return {k: self._serialize_metadata(v) for k, v in obj.items()}
|
return {k: self._serialize_metadata(v) for k, v in obj.items()}
|
||||||
@ -589,8 +589,7 @@ class KnowledgeBase(ABC):
|
|||||||
|
|
||||||
# 原子性写入(使用临时文件)
|
# 原子性写入(使用临时文件)
|
||||||
with tempfile.NamedTemporaryFile(
|
with tempfile.NamedTemporaryFile(
|
||||||
mode='w', dir=os.path.dirname(meta_file),
|
mode="w", dir=os.path.dirname(meta_file), prefix=".tmp_", suffix=".json", delete=False
|
||||||
prefix='.tmp_', suffix='.json', delete=False
|
|
||||||
) as tmp_file:
|
) as tmp_file:
|
||||||
json.dump(data, tmp_file, ensure_ascii=False, indent=2)
|
json.dump(data, tmp_file, ensure_ascii=False, indent=2)
|
||||||
temp_path = tmp_file.name
|
temp_path = tmp_file.name
|
||||||
|
|||||||
@ -299,7 +299,7 @@ class GraphDatabase:
|
|||||||
logger.info(f"Adding entity to {kgdb_name}")
|
logger.info(f"Adding entity to {kgdb_name}")
|
||||||
session.execute_write(_create_graph, triples)
|
session.execute_write(_create_graph, triples)
|
||||||
logger.info(f"Creating vector index for {kgdb_name} with {config.embed_model}")
|
logger.info(f"Creating vector index for {kgdb_name} with {config.embed_model}")
|
||||||
session.execute_write(_create_vector_index, getattr(cur_embed_info, 'dimension', 1024))
|
session.execute_write(_create_vector_index, getattr(cur_embed_info, "dimension", 1024))
|
||||||
|
|
||||||
# 收集所有需要处理的实体名称,去重
|
# 收集所有需要处理的实体名称,去重
|
||||||
all_entities = []
|
all_entities = []
|
||||||
|
|||||||
@ -72,11 +72,11 @@ class ChromaKB(KnowledgeBase):
|
|||||||
logger.info(f"Retrieved existing collection: {collection_name}")
|
logger.info(f"Retrieved existing collection: {collection_name}")
|
||||||
|
|
||||||
# 检查现有集合的配置是否匹配当前的 embed_info
|
# 检查现有集合的配置是否匹配当前的 embed_info
|
||||||
expected_model = getattr(embed_info, 'name', None) if embed_info else None
|
expected_model = getattr(embed_info, "name", None) if embed_info else None
|
||||||
if expected_model is None and hasattr(embed_info, 'get'):
|
if expected_model is None and hasattr(embed_info, "get"):
|
||||||
expected_model = embed_info.get('name')
|
expected_model = embed_info.get("name")
|
||||||
elif embed_info and isinstance(embed_info, dict):
|
elif embed_info and isinstance(embed_info, dict):
|
||||||
expected_model = embed_info.get('name')
|
expected_model = embed_info.get("name")
|
||||||
expected_model = expected_model or "default"
|
expected_model = expected_model or "default"
|
||||||
collection_metadata = collection.metadata or {}
|
collection_metadata = collection.metadata or {}
|
||||||
current_model = collection_metadata.get("embedding_model", "unknown")
|
current_model = collection_metadata.get("embedding_model", "unknown")
|
||||||
@ -93,13 +93,13 @@ class ChromaKB(KnowledgeBase):
|
|||||||
|
|
||||||
except Exception:
|
except Exception:
|
||||||
# 创建新集合
|
# 创建新集合
|
||||||
model_name = getattr(embed_info, 'name', None) if embed_info else None
|
model_name = getattr(embed_info, "name", None) if embed_info else None
|
||||||
if model_name is None and hasattr(embed_info, 'get'):
|
if model_name is None and hasattr(embed_info, "get"):
|
||||||
model_name = embed_info.get('name')
|
model_name = embed_info.get("name")
|
||||||
elif embed_info and isinstance(embed_info, dict):
|
elif embed_info and isinstance(embed_info, dict):
|
||||||
model_name = embed_info.get('name')
|
model_name = embed_info.get("name")
|
||||||
|
|
||||||
model_name = model_name or 'default'
|
model_name = model_name or "default"
|
||||||
logger.info(f"Creating new collection with embedding model: {model_name}")
|
logger.info(f"Creating new collection with embedding model: {model_name}")
|
||||||
collection_metadata = {
|
collection_metadata = {
|
||||||
"db_id": db_id,
|
"db_id": db_id,
|
||||||
|
|||||||
@ -103,7 +103,7 @@ class MilvusKB(KnowledgeBase):
|
|||||||
|
|
||||||
# 检查嵌入模型是否匹配
|
# 检查嵌入模型是否匹配
|
||||||
description = collection.description
|
description = collection.description
|
||||||
expected_model = getattr(embed_info, 'name', 'default') if embed_info else "default"
|
expected_model = getattr(embed_info, "name", "default") if embed_info else "default"
|
||||||
|
|
||||||
if expected_model not in description:
|
if expected_model not in description:
|
||||||
logger.warning(f"Collection {collection_name} model mismatch, recreating...")
|
logger.warning(f"Collection {collection_name} model mismatch, recreating...")
|
||||||
@ -116,8 +116,8 @@ class MilvusKB(KnowledgeBase):
|
|||||||
|
|
||||||
except Exception:
|
except Exception:
|
||||||
# 创建新集合
|
# 创建新集合
|
||||||
embedding_dim = getattr(embed_info, 'dimension', 1024) if embed_info else 1024
|
embedding_dim = getattr(embed_info, "dimension", 1024) if embed_info else 1024
|
||||||
model_name = getattr(embed_info, 'name', 'default') if embed_info else "default"
|
model_name = getattr(embed_info, "name", "default") if embed_info else "default"
|
||||||
|
|
||||||
# 定义集合Schema
|
# 定义集合Schema
|
||||||
fields = [
|
fields = [
|
||||||
|
|||||||
@ -2,7 +2,6 @@ import asyncio
|
|||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
||||||
from langchain_community.document_loaders import (
|
from langchain_community.document_loaders import (
|
||||||
CSVLoader,
|
CSVLoader,
|
||||||
JSONLoader,
|
JSONLoader,
|
||||||
@ -12,10 +11,10 @@ from langchain_community.document_loaders import (
|
|||||||
UnstructuredMarkdownLoader,
|
UnstructuredMarkdownLoader,
|
||||||
UnstructuredWordDocumentLoader,
|
UnstructuredWordDocumentLoader,
|
||||||
)
|
)
|
||||||
|
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
||||||
|
|
||||||
from src.utils import logger
|
from src.utils import logger
|
||||||
|
|
||||||
|
|
||||||
SUPPORTED_FILE_EXTENSIONS: tuple[str, ...] = (
|
SUPPORTED_FILE_EXTENSIONS: tuple[str, ...] = (
|
||||||
".txt",
|
".txt",
|
||||||
".md",
|
".md",
|
||||||
|
|||||||
@ -104,16 +104,11 @@ class KnowledgeBaseManager:
|
|||||||
shutil.copy2(meta_file, backup_file)
|
shutil.copy2(meta_file, backup_file)
|
||||||
|
|
||||||
# 准备数据
|
# 准备数据
|
||||||
data = {
|
data = {"databases": self.global_databases_meta, "updated_at": utc_isoformat(), "version": "2.0"}
|
||||||
"databases": self.global_databases_meta,
|
|
||||||
"updated_at": utc_isoformat(),
|
|
||||||
"version": "2.0"
|
|
||||||
}
|
|
||||||
|
|
||||||
# 原子性写入(使用临时文件)
|
# 原子性写入(使用临时文件)
|
||||||
with tempfile.NamedTemporaryFile(
|
with tempfile.NamedTemporaryFile(
|
||||||
mode='w', dir=os.path.dirname(meta_file),
|
mode="w", dir=os.path.dirname(meta_file), prefix=".tmp_", suffix=".json", delete=False
|
||||||
prefix='.tmp_', suffix='.json', delete=False
|
|
||||||
) as tmp_file:
|
) as tmp_file:
|
||||||
json.dump(data, tmp_file, ensure_ascii=False, indent=2)
|
json.dump(data, tmp_file, ensure_ascii=False, indent=2)
|
||||||
temp_path = tmp_file.name
|
temp_path = tmp_file.name
|
||||||
@ -520,7 +515,7 @@ class KnowledgeBaseManager:
|
|||||||
"chroma": {"missing_collections": [], "missing_files": []},
|
"chroma": {"missing_collections": [], "missing_files": []},
|
||||||
"milvus": {"missing_collections": [], "missing_files": []},
|
"milvus": {"missing_collections": [], "missing_files": []},
|
||||||
"total_missing_collections": 0,
|
"total_missing_collections": 0,
|
||||||
"total_missing_files": 0
|
"total_missing_files": 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info("开始检测向量数据库与元数据的一致性...")
|
logger.info("开始检测向量数据库与元数据的一致性...")
|
||||||
@ -573,10 +568,7 @@ class KnowledgeBaseManager:
|
|||||||
if not collection_name.startswith("kb_"):
|
if not collection_name.startswith("kb_"):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
collection_info = {
|
collection_info = {"collection_name": collection_name, "detected_at": utc_isoformat()}
|
||||||
"collection_name": collection_name,
|
|
||||||
"detected_at": utc_isoformat()
|
|
||||||
}
|
|
||||||
|
|
||||||
# 尝试获取集合的基本信息
|
# 尝试获取集合的基本信息
|
||||||
try:
|
try:
|
||||||
@ -588,7 +580,10 @@ class KnowledgeBaseManager:
|
|||||||
collection_info["count"] = "unknown"
|
collection_info["count"] = "unknown"
|
||||||
|
|
||||||
inconsistencies["missing_collections"].append(collection_info)
|
inconsistencies["missing_collections"].append(collection_info)
|
||||||
logger.warning(f"发现 ChromaDB 中存在但 metadata 中缺失的集合: {collection_name} (文档数: {collection_info['count']})")
|
logger.warning(
|
||||||
|
f"发现 ChromaDB 中存在但 metadata 中缺失的集合: {collection_name} "
|
||||||
|
f"(文档数: {collection_info['count']})"
|
||||||
|
)
|
||||||
|
|
||||||
# 检查文件级别的不一致(针对已知的数据库)
|
# 检查文件级别的不一致(针对已知的数据库)
|
||||||
for db_id in metadata_collection_names:
|
for db_id in metadata_collection_names:
|
||||||
@ -597,18 +592,23 @@ class KnowledgeBaseManager:
|
|||||||
actual_count = collection.count()
|
actual_count = collection.count()
|
||||||
|
|
||||||
# 获取 metadata 中记录的文件数量
|
# 获取 metadata 中记录的文件数量
|
||||||
metadata_files_count = sum(1 for file_info in chroma_kb.files_meta.values()
|
metadata_files_count = sum(
|
||||||
if file_info.get("database_id") == db_id)
|
1 for file_info in chroma_kb.files_meta.values() if file_info.get("database_id") == db_id
|
||||||
|
)
|
||||||
|
|
||||||
# 如果向量数据库中有数据但 metadata 中没有文件记录,可能存在文件缺失
|
# 如果向量数据库中有数据但 metadata 中没有文件记录,可能存在文件缺失
|
||||||
if actual_count > 0 and metadata_files_count == 0:
|
if actual_count > 0 and metadata_files_count == 0:
|
||||||
inconsistencies["missing_files"].append({
|
inconsistencies["missing_files"].append(
|
||||||
"database_id": db_id,
|
{
|
||||||
"vector_count": actual_count,
|
"database_id": db_id,
|
||||||
"metadata_files_count": metadata_files_count,
|
"vector_count": actual_count,
|
||||||
"detected_at": utc_isoformat()
|
"metadata_files_count": metadata_files_count,
|
||||||
})
|
"detected_at": utc_isoformat(),
|
||||||
logger.warning(f"发现数据库 {db_id} 在 ChromaDB 中有 {actual_count} 条向量数据,但 metadata 中没有文件记录")
|
}
|
||||||
|
)
|
||||||
|
logger.warning(
|
||||||
|
f"发现数据库 {db_id} 在 ChromaDB 中有 {actual_count} 条向量数据,但 metadata 中没有文件记录"
|
||||||
|
)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug(f"检查数据库 {db_id} 的文件一致性时出错: {e}")
|
logger.debug(f"检查数据库 {db_id} 的文件一致性时出错: {e}")
|
||||||
@ -640,14 +640,12 @@ class KnowledgeBaseManager:
|
|||||||
if not collection_name.startswith("kb_"):
|
if not collection_name.startswith("kb_"):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
collection_info = {
|
collection_info = {"collection_name": collection_name, "detected_at": utc_isoformat()}
|
||||||
"collection_name": collection_name,
|
|
||||||
"detected_at": utc_isoformat()
|
|
||||||
}
|
|
||||||
|
|
||||||
# 尝试获取集合的基本信息
|
# 尝试获取集合的基本信息
|
||||||
try:
|
try:
|
||||||
from pymilvus import Collection
|
from pymilvus import Collection
|
||||||
|
|
||||||
collection = Collection(name=collection_name, using=milvus_kb.connection_alias)
|
collection = Collection(name=collection_name, using=milvus_kb.connection_alias)
|
||||||
collection_info["count"] = collection.num_entities
|
collection_info["count"] = collection.num_entities
|
||||||
collection_info["description"] = collection.description
|
collection_info["description"] = collection.description
|
||||||
@ -656,30 +654,39 @@ class KnowledgeBaseManager:
|
|||||||
collection_info["count"] = "unknown"
|
collection_info["count"] = "unknown"
|
||||||
|
|
||||||
inconsistencies["missing_collections"].append(collection_info)
|
inconsistencies["missing_collections"].append(collection_info)
|
||||||
logger.warning(f"发现 Milvus 中存在但 metadata 中缺失的集合: {collection_name} (实体数: {collection_info['count']})")
|
logger.warning(
|
||||||
|
f"发现 Milvus 中存在但 metadata 中缺失的集合: {collection_name} "
|
||||||
|
f"(实体数: {collection_info['count']})"
|
||||||
|
)
|
||||||
|
|
||||||
# 检查文件级别的不一致(针对已知的数据库)
|
# 检查文件级别的不一致(针对已知的数据库)
|
||||||
for db_id in metadata_collection_names:
|
for db_id in metadata_collection_names:
|
||||||
try:
|
try:
|
||||||
if utility.has_collection(db_id, using=milvus_kb.connection_alias):
|
if utility.has_collection(db_id, using=milvus_kb.connection_alias):
|
||||||
from pymilvus import Collection
|
from pymilvus import Collection
|
||||||
|
|
||||||
collection = Collection(name=db_id, using=milvus_kb.connection_alias)
|
collection = Collection(name=db_id, using=milvus_kb.connection_alias)
|
||||||
actual_count = collection.num_entities
|
actual_count = collection.num_entities
|
||||||
|
|
||||||
# 获取 metadata 中记录的文件数量
|
# 获取 metadata 中记录的文件数量
|
||||||
metadata_files_count = sum(1 for file_info in milvus_kb.files_meta.values()
|
metadata_files_count = sum(
|
||||||
if file_info.get("database_id") == db_id)
|
1 for file_info in milvus_kb.files_meta.values() if file_info.get("database_id") == db_id
|
||||||
|
)
|
||||||
|
|
||||||
# 如果向量数据库中有数据但 metadata 中没有文件记录,可能存在文件缺失
|
# 如果向量数据库中有数据但 metadata 中没有文件记录,可能存在文件缺失
|
||||||
if actual_count > 0 and metadata_files_count == 0:
|
if actual_count > 0 and metadata_files_count == 0:
|
||||||
inconsistencies["missing_files"].append({
|
inconsistencies["missing_files"].append(
|
||||||
"database_id": db_id,
|
{
|
||||||
"vector_count": actual_count,
|
"database_id": db_id,
|
||||||
"metadata_files_count": metadata_files_count,
|
"vector_count": actual_count,
|
||||||
"detected_at": utc_isoformat()
|
"metadata_files_count": metadata_files_count,
|
||||||
})
|
"detected_at": utc_isoformat(),
|
||||||
logger.warning(f"发现数据库 {db_id} 在 Milvus 中有 {actual_count} 条向量数据,但 metadata 中没有文件记录")
|
}
|
||||||
|
)
|
||||||
|
logger.warning(
|
||||||
|
f"发现数据库 {db_id} 在 Milvus 中有 {actual_count} 条向量数据,"
|
||||||
|
"但 metadata 中没有文件记录"
|
||||||
|
)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug(f"检查数据库 {db_id} 的文件一致性时出错: {e}")
|
logger.debug(f"检查数据库 {db_id} 的文件一致性时出错: {e}")
|
||||||
@ -706,25 +713,31 @@ class KnowledgeBaseManager:
|
|||||||
chroma_missing = inconsistencies["chroma"]["missing_collections"]
|
chroma_missing = inconsistencies["chroma"]["missing_collections"]
|
||||||
chroma_files_missing = inconsistencies["chroma"]["missing_files"]
|
chroma_files_missing = inconsistencies["chroma"]["missing_files"]
|
||||||
if chroma_missing or chroma_files_missing:
|
if chroma_missing or chroma_files_missing:
|
||||||
logger.warning(f"ChromaDB 不一致情况:")
|
logger.warning("ChromaDB 不一致情况:")
|
||||||
logger.warning(f" 缺失集合数量: {len(chroma_missing)}")
|
logger.warning(f" 缺失集合数量: {len(chroma_missing)}")
|
||||||
for collection_info in chroma_missing:
|
for collection_info in chroma_missing:
|
||||||
logger.warning(f" - 集合: {collection_info['collection_name']}, 向量数: {collection_info['count']}")
|
logger.warning(f" - 集合: {collection_info['collection_name']}, 向量数: {collection_info['count']}")
|
||||||
logger.warning(f" 缺失文件记录数量: {len(chroma_files_missing)}")
|
logger.warning(f" 缺失文件记录数量: {len(chroma_files_missing)}")
|
||||||
for file_info in chroma_files_missing:
|
for file_info in chroma_files_missing:
|
||||||
logger.warning(f" - 数据库: {file_info['database_id']}, 向量数: {file_info['vector_count']}, 元数据文件数: {file_info['metadata_files_count']}")
|
logger.warning(
|
||||||
|
f" - 数据库: {file_info['database_id']}, 向量数: {file_info['vector_count']}, "
|
||||||
|
f"元数据文件数: {file_info['metadata_files_count']}"
|
||||||
|
)
|
||||||
|
|
||||||
# Milvus 不一致情况
|
# Milvus 不一致情况
|
||||||
milvus_missing = inconsistencies["milvus"]["missing_collections"]
|
milvus_missing = inconsistencies["milvus"]["missing_collections"]
|
||||||
milvus_files_missing = inconsistencies["milvus"]["missing_files"]
|
milvus_files_missing = inconsistencies["milvus"]["missing_files"]
|
||||||
if milvus_missing or milvus_files_missing:
|
if milvus_missing or milvus_files_missing:
|
||||||
logger.warning(f"Milvus 不一致情况:")
|
logger.warning("Milvus 不一致情况:")
|
||||||
logger.warning(f" 缺失集合数量: {len(milvus_missing)}")
|
logger.warning(f" 缺失集合数量: {len(milvus_missing)}")
|
||||||
for collection_info in milvus_missing:
|
for collection_info in milvus_missing:
|
||||||
logger.warning(f" - 集合: {collection_info['collection_name']}, 实体数: {collection_info['count']}")
|
logger.warning(f" - 集合: {collection_info['collection_name']}, 实体数: {collection_info['count']}")
|
||||||
logger.warning(f" 缺失文件记录数量: {len(milvus_files_missing)}")
|
logger.warning(f" 缺失文件记录数量: {len(milvus_files_missing)}")
|
||||||
for file_info in milvus_files_missing:
|
for file_info in milvus_files_missing:
|
||||||
logger.warning(f" - 数据库: {file_info['database_id']}, 向量数: {file_info['vector_count']}, 元数据文件数: {file_info['metadata_files_count']}")
|
logger.warning(
|
||||||
|
f" - 数据库: {file_info['database_id']}, 向量数: {file_info['vector_count']}, "
|
||||||
|
f"元数据文件数: {file_info['metadata_files_count']}"
|
||||||
|
)
|
||||||
|
|
||||||
logger.warning("=" * 80)
|
logger.warning("=" * 80)
|
||||||
logger.warning(f"总计:缺失集合 {total_missing_collections} 个,缺失文件记录 {total_missing_files} 个")
|
logger.warning(f"总计:缺失集合 {total_missing_collections} 个,缺失文件记录 {total_missing_files} 个")
|
||||||
|
|||||||
@ -211,7 +211,7 @@ def get_embedding_config(embed_info: dict) -> dict:
|
|||||||
try:
|
try:
|
||||||
if embed_info:
|
if embed_info:
|
||||||
# 处理 embed_info 可能是字典或 EmbedModelInfo 对象的情况
|
# 处理 embed_info 可能是字典或 EmbedModelInfo 对象的情况
|
||||||
if hasattr(embed_info, 'name'):
|
if hasattr(embed_info, "name"):
|
||||||
# EmbedModelInfo 对象
|
# EmbedModelInfo 对象
|
||||||
config_dict["model"] = embed_info.name
|
config_dict["model"] = embed_info.name
|
||||||
config_dict["api_key"] = os.getenv(embed_info.api_key, embed_info.api_key)
|
config_dict["api_key"] = os.getenv(embed_info.api_key, embed_info.api_key)
|
||||||
|
|||||||
@ -12,7 +12,8 @@ from src.storage.db.models import Conversation, ConversationStats, Message, Tool
|
|||||||
from src.utils import logger
|
from src.utils import logger
|
||||||
from src.utils.datetime_utils import utc_now
|
from src.utils.datetime_utils import utc_now
|
||||||
|
|
||||||
#TODO:[未完成]待修改为异步版本
|
# TODO:[未完成]待修改为异步版本
|
||||||
|
|
||||||
|
|
||||||
class ConversationManager:
|
class ConversationManager:
|
||||||
"""Manager for conversation storage operations"""
|
"""Manager for conversation storage operations"""
|
||||||
|
|||||||
@ -22,6 +22,7 @@ except ImportError:
|
|||||||
# TODO:[优化建议]需要将数据库修改为异步的aiosqlite或者异步mysql,缓存使用Redis存储
|
# TODO:[优化建议]需要将数据库修改为异步的aiosqlite或者异步mysql,缓存使用Redis存储
|
||||||
# TODO:[已完成]为DBManager添加单例模式
|
# TODO:[已完成]为DBManager添加单例模式
|
||||||
|
|
||||||
|
|
||||||
class DBManager(metaclass=SingletonMeta):
|
class DBManager(metaclass=SingletonMeta):
|
||||||
"""数据库管理器 - 只提供基础的数据库连接和会话管理"""
|
"""数据库管理器 - 只提供基础的数据库连接和会话管理"""
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user