feat: 更新知识库相关配置
- 实现QA数据 - 优化现在创建知识库的体验 - 优化文件处理与加载方法 - 整体的格式检查与优化 - 优化 Embedding model 的加载逻辑,修复并行问题 - 优化整体的颜色布局 - 移除未使用的接口(前后端) - 优化知识库的 chunk 逻辑 - 添加新的 Embedding模型支持 - 修复新建知识库后,Agent无法reload的问题
This commit is contained in:
parent
9f5924b8e6
commit
7d4722b62c
1
.gitignore
vendored
1
.gitignore
vendored
@ -28,6 +28,7 @@ cache
|
||||
### IDE
|
||||
.vscode
|
||||
.idea
|
||||
.vibe
|
||||
*.nogit*
|
||||
*.private*
|
||||
*.local*
|
||||
|
||||
@ -8,7 +8,8 @@
|
||||
- [x] 集成 MinerU 处理 PDF 文件。(🌟🌟🌟)
|
||||
- [x] 优化现有向量模型的逻辑,全局配置的向量模型作为新建知识库的默认模型,但是查询的时候,使用对应数据库的向量模型查询(单例模式)(🌟🌟🌟)
|
||||
- [x] 多类型知识库支持(🌟🌟🌟🌟🌟)
|
||||
- [ ] 设计问答知识库,支持针对问答对的调优
|
||||
- [x] 设计问答知识库,支持针对问答对的调优
|
||||
- [ ] 进一步优化知识库的设计,需要将 chroma 和 milvus 知识库合并在一起,只是说使用的后端存储不同,这个应该放在创建知识库的时候选择,如果选择的是 CommonRAG 知识库的话就是用这种。
|
||||
|
||||
🐛**BUGs**
|
||||
- [x] 智能体的工具切换有问题
|
||||
|
||||
@ -18,6 +18,7 @@ dependencies = [
|
||||
"langchain-mcp-adapters>=0.1.9",
|
||||
"langchain-openai>=0.3.14",
|
||||
"langchain-tavily>=0.2.3",
|
||||
"langchain-text-splitters>=0.3.9",
|
||||
"langchain-together>=0.3.0",
|
||||
"langgraph>=0.3.34",
|
||||
"langgraph-checkpoint-sqlite>=2.0.7",
|
||||
@ -26,6 +27,7 @@ dependencies = [
|
||||
"lightrag-hku>=1.3.9",
|
||||
"llama-index>=0.12.33",
|
||||
"llama-index-readers-file>=0.4.7",
|
||||
"markdownify>=1.1.0",
|
||||
"mcp>=1.12.0",
|
||||
"mineru>=2.0.6",
|
||||
"neo4j>=5.28.1",
|
||||
@ -36,6 +38,7 @@ dependencies = [
|
||||
"pyjwt>=2.8.0",
|
||||
"pymilvus>=2.5.8",
|
||||
"pymupdf>=1.25.5",
|
||||
"python-docx>=1.2.0",
|
||||
"python-dotenv>=1.1.0",
|
||||
"python-jose[cryptography]>=3.4.0",
|
||||
"python-multipart>=0.0.20",
|
||||
@ -54,10 +57,11 @@ lint.select = [ # 选择的规则
|
||||
"W",
|
||||
"UP",
|
||||
]
|
||||
lint.ignore = ["F401"] # 忽略的规则
|
||||
lint.ignore = ["F401", "E501"] # 忽略的规则
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"ipykernel>=6.30.0",
|
||||
"ruff>=0.12.1",
|
||||
"vllm>=0.8.5.post1",
|
||||
]
|
||||
|
||||
@ -364,12 +364,3 @@ async def add_neo4j_entities(
|
||||
"message": f"添加实体失败: {e}",
|
||||
"status": "failed"
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# === 兼容性接口 (保持向后兼容) ===
|
||||
# =============================================================================
|
||||
|
||||
@graph.get("/graph")
|
||||
async def get_graph_info_compat(current_user: User = Depends(get_admin_user)):
|
||||
"""兼容性接口:获取图数据库信息"""
|
||||
return await get_graph_info(current_user)
|
||||
|
||||
@ -30,18 +30,25 @@ async def create_database(
|
||||
description: str = Body(...),
|
||||
embed_model_name: str = Body(...),
|
||||
kb_type: str = Body("lightrag"),
|
||||
additional_params: dict = Body({}),
|
||||
current_user: User = Depends(get_admin_user)
|
||||
):
|
||||
"""创建知识库"""
|
||||
logger.debug(f"Create database {database_name} with kb_type {kb_type}")
|
||||
logger.debug(f"Create database {database_name} with kb_type {kb_type}, additional_params {additional_params}")
|
||||
try:
|
||||
embed_info = config.embed_model_names[embed_model_name]
|
||||
database_info = knowledge_base.create_database(
|
||||
database_name,
|
||||
description,
|
||||
kb_type=kb_type,
|
||||
embed_info=embed_info
|
||||
embed_info=embed_info,
|
||||
**additional_params
|
||||
)
|
||||
|
||||
# 需要重新加载所有智能体,因为工具刷新了
|
||||
from src.agents import agent_manager
|
||||
await agent_manager.reload_all()
|
||||
|
||||
return database_info
|
||||
except Exception as e:
|
||||
logger.error(f"创建数据库失败 {e}, {traceback.format_exc()}")
|
||||
@ -77,6 +84,11 @@ async def delete_database(db_id: str, current_user: User = Depends(get_admin_use
|
||||
logger.debug(f"Delete database {db_id}")
|
||||
try:
|
||||
knowledge_base.delete_database(db_id)
|
||||
|
||||
# 需要重新加载所有智能体,因为工具刷新了
|
||||
from src.agents import agent_manager
|
||||
await agent_manager.reload_all()
|
||||
|
||||
return {"message": "删除成功"}
|
||||
except Exception as e:
|
||||
logger.error(f"删除数据库失败 {e}, {traceback.format_exc()}")
|
||||
|
||||
@ -199,7 +199,7 @@ async def check_ocr_services_health(current_user: User = Depends(get_admin_user)
|
||||
if os.path.exists(model_dir) and os.path.exists(det_model_path) and os.path.exists(rec_model_path):
|
||||
# 尝试初始化RapidOCR
|
||||
from rapidocr_onnxruntime import RapidOCR
|
||||
test_ocr = RapidOCR(det_box_thresh=0.3, det_model_path=det_model_path, rec_model_path=rec_model_path)
|
||||
test_ocr = RapidOCR(det_box_thresh=0.3, det_model_path=det_model_path, rec_model_path=rec_model_path) # noqa: F841
|
||||
health_status["rapid_ocr"]["status"] = "healthy"
|
||||
health_status["rapid_ocr"]["message"] = "RapidOCR模型已加载"
|
||||
else:
|
||||
|
||||
@ -12,4 +12,4 @@ async def get_tools(current_user: User = Depends(get_required_user)):
|
||||
tools_info = get_all_tools_info()
|
||||
return {"tools": tools_info}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
return {"error": str(e)}
|
||||
|
||||
@ -17,10 +17,6 @@ from src.knowledge.chroma_kb import ChromaKB # noqa: E402
|
||||
from src.knowledge.milvus_kb import MilvusKB # noqa: E402
|
||||
|
||||
# 注册知识库类型
|
||||
KnowledgeBaseFactory.register("lightrag", LightRagKB, {
|
||||
"description": "基于图检索的知识库,支持实体关系构建和复杂查询"
|
||||
})
|
||||
|
||||
KnowledgeBaseFactory.register("chroma", ChromaKB, {
|
||||
"chunk_size": 1000,
|
||||
"chunk_overlap": 200,
|
||||
@ -33,6 +29,12 @@ KnowledgeBaseFactory.register("milvus", MilvusKB, {
|
||||
"description": "基于 Milvus 的生产级向量知识库,适合大规模高性能部署"
|
||||
})
|
||||
|
||||
|
||||
KnowledgeBaseFactory.register("lightrag", LightRagKB, {
|
||||
"description": "基于图检索的知识库,支持实体关系构建和复杂查询"
|
||||
})
|
||||
|
||||
|
||||
# 创建知识库管理器
|
||||
work_dir = os.path.join(config.save_dir, "knowledge_base_data")
|
||||
knowledge_base = KnowledgeBaseManager(work_dir)
|
||||
|
||||
@ -16,9 +16,9 @@ class AgentManager:
|
||||
for agent_id in self._classes.keys():
|
||||
self.get_agent(agent_id)
|
||||
|
||||
def get_agent(self, agent_id, **kwargs):
|
||||
def get_agent(self, agent_id, reload=False, **kwargs):
|
||||
# 检查是否已经创建了该 agent 的实例
|
||||
if agent_id not in self._instances:
|
||||
if reload or agent_id not in self._instances:
|
||||
agent_class = self._classes[agent_id]
|
||||
self._instances[agent_id] = agent_class()
|
||||
|
||||
@ -27,6 +27,10 @@ class AgentManager:
|
||||
def get_agents(self):
|
||||
return list(self._instances.values())
|
||||
|
||||
async def reload_all(self):
|
||||
for agent_id in self._classes.keys():
|
||||
self.get_agent(agent_id, reload=True)
|
||||
|
||||
async def get_agents_info(self):
|
||||
agents = self.get_agents()
|
||||
return await asyncio.gather(*[a.get_info() for a in agents])
|
||||
|
||||
@ -65,9 +65,14 @@ class ChatbotAgent(BaseAgent):
|
||||
if self.graph:
|
||||
return self.graph
|
||||
|
||||
runnable_tools = get_runnable_tools()
|
||||
tools = list(runnable_tools.values())
|
||||
tools_name = list(runnable_tools.keys())
|
||||
logger.debug(f"build graph `{self.id}` with tools: {tools_name}")
|
||||
|
||||
workflow = StateGraph(State, config_schema=self.config_schema)
|
||||
workflow.add_node("chatbot", self.llm_call)
|
||||
workflow.add_node("tools", ToolNode(tools=list(get_runnable_tools().values())))
|
||||
workflow.add_node("tools", ToolNode(tools=tools))
|
||||
workflow.add_edge(START, "chatbot")
|
||||
workflow.add_conditional_edges(
|
||||
"chatbot",
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Any, List, Optional
|
||||
from typing import Any, Optional
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
import os
|
||||
from enum import Enum
|
||||
@ -14,17 +14,17 @@ class SearchAPI(Enum):
|
||||
NONE = "none"
|
||||
|
||||
class MCPConfig(BaseModel):
|
||||
url: Optional[str] = Field(
|
||||
url: str | None = Field(
|
||||
default=None,
|
||||
optional=True,
|
||||
)
|
||||
"""The URL of the MCP server"""
|
||||
tools: Optional[List[str]] = Field(
|
||||
tools: list[str] | None = Field(
|
||||
default=None,
|
||||
optional=True,
|
||||
)
|
||||
"""The tools to make available to the LLM"""
|
||||
auth_required: Optional[bool] = Field(
|
||||
auth_required: bool | None = Field(
|
||||
default=False,
|
||||
optional=True,
|
||||
)
|
||||
@ -63,7 +63,7 @@ class OriConfiguration(BaseModel):
|
||||
"min": 1,
|
||||
"max": 20,
|
||||
"step": 1,
|
||||
"description": "Maximum number of research units to run concurrently. This will allow the researcher to use multiple sub-agents to conduct research. Note: with more concurrency, you may run into rate limits."
|
||||
"description": "Maximum number of research units to run concurrently. This will allow the researcher to use multiple sub-agents to conduct research. Note: with more concurrency, you may run into rate limits." # noqa: E501
|
||||
}
|
||||
}
|
||||
)
|
||||
@ -192,7 +192,7 @@ class OriConfiguration(BaseModel):
|
||||
}
|
||||
)
|
||||
# MCP server configuration
|
||||
mcp_config: Optional[MCPConfig] = Field(
|
||||
mcp_config: MCPConfig | None = Field(
|
||||
default=None,
|
||||
optional=True,
|
||||
metadata={
|
||||
@ -202,7 +202,7 @@ class OriConfiguration(BaseModel):
|
||||
}
|
||||
}
|
||||
)
|
||||
mcp_prompt: Optional[str] = Field(
|
||||
mcp_prompt: str | None = Field(
|
||||
default=None,
|
||||
optional=True,
|
||||
metadata={
|
||||
@ -216,7 +216,7 @@ class OriConfiguration(BaseModel):
|
||||
|
||||
@classmethod
|
||||
def from_runnable_config(
|
||||
cls, config: Optional[RunnableConfig] = None
|
||||
cls, config: RunnableConfig | None = None
|
||||
) -> "Configuration":
|
||||
"""Create a Configuration instance from a RunnableConfig."""
|
||||
configurable = config.get("configurable", {}) if config else {}
|
||||
@ -232,4 +232,4 @@ class OriConfiguration(BaseModel):
|
||||
|
||||
|
||||
class Configuration(BaseModelConfiguration, OriConfiguration):
|
||||
pass
|
||||
pass
|
||||
|
||||
@ -245,7 +245,7 @@ async def researcher_tools(state: ResearcherState, config: RunnableConfig) -> Co
|
||||
most_recent_message = researcher_messages[-1]
|
||||
# Early Exit Criteria: No tool calls (or native web search calls)were made by the researcher
|
||||
if not most_recent_message.tool_calls and not (openai_websearch_called(most_recent_message) or anthropic_websearch_called(most_recent_message)):
|
||||
logger.info(f"researcher_tools: no tool calls")
|
||||
logger.info("researcher_tools: no tool calls")
|
||||
return Command(
|
||||
goto="compress_research",
|
||||
)
|
||||
@ -264,14 +264,14 @@ async def researcher_tools(state: ResearcherState, config: RunnableConfig) -> Co
|
||||
# Late Exit Criteria: We have exceeded our max guardrail tool call iterations or the most recent message contains a ResearchComplete tool call
|
||||
# These are late exit criteria because we need to add ToolMessages
|
||||
if state.get("tool_call_iterations", 0) >= configurable.max_react_tool_calls or any(tool_call["name"] == "ResearchComplete" for tool_call in most_recent_message.tool_calls):
|
||||
logger.info(f"researcher_tools: max_react_tool_calls or ResearchComplete")
|
||||
logger.info("researcher_tools: max_react_tool_calls or ResearchComplete")
|
||||
return Command(
|
||||
goto="compress_research",
|
||||
update={
|
||||
"researcher_messages": tool_outputs,
|
||||
}
|
||||
)
|
||||
logger.info(f"researcher_tools: goto researcher")
|
||||
logger.info("researcher_tools: goto researcher")
|
||||
return Command(
|
||||
goto="researcher",
|
||||
update={
|
||||
@ -309,7 +309,7 @@ async def compress_research(state: ResearcherState, config: RunnableConfig):
|
||||
print(f"Token limit exceeded while synthesizing: {e}. Pruning the messages to try again.")
|
||||
continue
|
||||
print(f"Error synthesizing research report: {e}")
|
||||
logger.info(f"compress_research: Error synthesizing research report: Maximum retries exceeded")
|
||||
logger.info("compress_research: Error synthesizing research report: Maximum retries exceeded")
|
||||
return {
|
||||
"compressed_research": "Error synthesizing research report: Maximum retries exceeded",
|
||||
"raw_notes": ["\n".join([str(m.content) for m in filter_messages(researcher_messages, include_types=["tool", "ai"])])]
|
||||
@ -376,7 +376,7 @@ async def final_report_generation(state: AgentState, config: RunnableConfig):
|
||||
"final_report": f"Error generating final report: {e}",
|
||||
**cleared_state
|
||||
}
|
||||
logger.info(f"final_report_generation: Error generating final report: Maximum retries exceeded")
|
||||
logger.info("final_report_generation: Error generating final report: Maximum retries exceeded")
|
||||
return {
|
||||
"final_report": "Error generating final report: Maximum retries exceeded",
|
||||
"messages": [final_report],
|
||||
@ -392,4 +392,4 @@ deep_researcher_builder.add_edge(START, "clarify_with_user")
|
||||
deep_researcher_builder.add_edge("research_supervisor", "final_report_generation")
|
||||
deep_researcher_builder.add_edge("final_report_generation", END)
|
||||
|
||||
deep_researcher = deep_researcher_builder.compile(checkpointer=InMemorySaver())
|
||||
deep_researcher = deep_researcher_builder.compile(checkpointer=InMemorySaver())
|
||||
|
||||
@ -39,7 +39,7 @@ For the verification message when no clarification is needed:
|
||||
"""
|
||||
|
||||
|
||||
transform_messages_into_research_topic_prompt = """You will be given a set of messages that have been exchanged so far between yourself and the user.
|
||||
transform_messages_into_research_topic_prompt = """You will be given a set of messages that have been exchanged so far between yourself and the user.
|
||||
Your job is to translate these messages into a more detailed and concrete research question that will be used to guide the research.
|
||||
|
||||
The messages that have been exchanged so far between yourself and the user are:
|
||||
@ -78,12 +78,12 @@ Guidelines:
|
||||
lead_researcher_prompt = """You are a research supervisor. Your job is to conduct research by calling the "ConductResearch" tool. For context, today's date is {date}.
|
||||
|
||||
<Task>
|
||||
Your focus is to call the "ConductResearch" tool to conduct research against the overall research question passed in by the user.
|
||||
Your focus is to call the "ConductResearch" tool to conduct research against the overall research question passed in by the user.
|
||||
When you are completely satisfied with the research findings returned from the tool calls, then you should call the "ResearchComplete" tool to indicate that you are done with your research.
|
||||
</Task>
|
||||
|
||||
<Instructions>
|
||||
1. When you start, you will be provided a research question from a user.
|
||||
1. When you start, you will be provided a research question from a user.
|
||||
2. You should immediately call the "ConductResearch" tool to conduct research for the research question. You can call the tool up to {max_concurrent_research_units} times in a single iteration.
|
||||
3. Each ConductResearch tool call will spawn a research agent dedicated to the specific topic that you pass in. You will get back a comprehensive report of research findings on that topic.
|
||||
4. Reason carefully about whether all of the returned research findings together are comprehensive enough for a detailed report to answer the overall research question.
|
||||
@ -101,13 +101,13 @@ When you are completely satisfied with the research findings returned from the t
|
||||
- Do not call the "ConductResearch" tool to synthesize information you have already gathered.
|
||||
|
||||
**Parallel research saves the user time, but reason carefully about when you should use it**
|
||||
- Calling the "ConductResearch" tool multiple times in parallel can save the user time.
|
||||
- Calling the "ConductResearch" tool multiple times in parallel can save the user time.
|
||||
- You should only call the "ConductResearch" tool multiple times in parallel if the different topics that you are researching can be researched independently in parallel with respect to the user's overall question.
|
||||
- This can be particularly helpful if the user is asking for a comparison of X and Y, if the user is asking for a list of entities that each can be researched independently, or if the user is asking for multiple perspectives on a topic.
|
||||
- Each research agent needs to be provided all of the context that is necessary to focus on a sub-topic.
|
||||
- Do not call the "ConductResearch" tool more than {max_concurrent_research_units} times at once. This limit is enforced by the user. It is perfectly fine, and expected, that you return less than this number of tool calls.
|
||||
- If you are not confident in how you can parallelize research, you can call the "ConductResearch" tool a single time on a more general topic in order to gather more background information, so you have more context later to reason about if it's necessary to parallelize research.
|
||||
- Each parallel "ConductResearch" linearly scales cost. The benefit of parallel research is that it can save the user time, but carefully think about whether the additional cost is worth the benefit.
|
||||
- Each parallel "ConductResearch" linearly scales cost. The benefit of parallel research is that it can save the user time, but carefully think about whether the additional cost is worth the benefit.
|
||||
- For example, if you could search three clear topics in parallel, or break them each into two more subtopics to do six total in parallel, you should think about whether splitting into smaller subtopics is worth the cost. The researchers are quite comprehensive, so it's possible that you could get the same information with less cost by only calling the "ConductResearch" tool three times in this case.
|
||||
- Also consider where there might be dependencies that cannot be parallelized. For example, if asked for details about some entities, you first need to find the entities before you can research them in detail in parallel.
|
||||
|
||||
@ -268,7 +268,7 @@ Make sure that your sections are cohesive, and make sense for the reader.
|
||||
For each section of the report, do the following:
|
||||
- Use simple, clear language
|
||||
- Use ## for section title (Markdown format) for each section of the report
|
||||
- Do NOT ever refer to yourself as the writer of the report. This should be a professional report without any self-referential language.
|
||||
- Do NOT ever refer to yourself as the writer of the report. This should be a professional report without any self-referential language.
|
||||
- Do not say what you are doing in the report. Just write the report without any commentary from yourself.
|
||||
|
||||
Format the report in clear markdown with proper structure and include source references where appropriate.
|
||||
@ -336,11 +336,11 @@ Example 2 (for a scientific article):
|
||||
```json
|
||||
{{
|
||||
"summary": "A new study published in Nature Climate Change reveals that global sea levels are rising faster than previously thought. Researchers analyzed satellite data from 1993 to 2022 and found that the rate of sea-level rise has accelerated by 0.08 mm/year² over the past three decades. This acceleration is primarily attributed to melting ice sheets in Greenland and Antarctica. The study projects that if current trends continue, global sea levels could rise by up to 2 meters by 2100, posing significant risks to coastal communities worldwide.",
|
||||
"key_excerpts": "Our findings indicate a clear acceleration in sea-level rise, which has significant implications for coastal planning and adaptation strategies, lead author Dr. Emily Brown stated. The rate of ice sheet melt in Greenland and Antarctica has tripled since the 1990s, the study reports. Without immediate and substantial reductions in greenhouse gas emissions, we are looking at potentially catastrophic sea-level rise by the end of this century, warned co-author Professor Michael Green."
|
||||
"key_excerpts": "Our findings indicate a clear acceleration in sea-level rise, which has significant implications for coastal planning and adaptation strategies, lead author Dr. Emily Brown stated. The rate of ice sheet melt in Greenland and Antarctica has tripled since the 1990s, the study reports. Without immediate and substantial reductions in greenhouse gas emissions, we are looking at potentially catastrophic sea-level rise by the end of this century, warned co-author Professor Michael Green."
|
||||
}}
|
||||
```
|
||||
|
||||
Remember, your goal is to create a summary that can be easily understood and utilized by a downstream research agent while preserving the most critical information from the original webpage.
|
||||
|
||||
Today's date is {date}.
|
||||
"""
|
||||
"""
|
||||
|
||||
@ -53,7 +53,7 @@ class AgentInputState(MessagesState):
|
||||
|
||||
class AgentState(MessagesState):
|
||||
supervisor_messages: Annotated[list[MessageLikeRepresentation], override_reducer]
|
||||
research_brief: Optional[str]
|
||||
research_brief: str | None
|
||||
raw_notes: Annotated[list[str], override_reducer] = []
|
||||
notes: Annotated[list[str], override_reducer] = []
|
||||
final_report: str
|
||||
@ -74,4 +74,4 @@ class ResearcherState(TypedDict):
|
||||
|
||||
class ResearcherOutputState(BaseModel):
|
||||
compressed_research: str
|
||||
raw_notes: Annotated[list[str], override_reducer] = []
|
||||
raw_notes: Annotated[list[str], override_reducer] = []
|
||||
|
||||
@ -3,8 +3,8 @@ import aiohttp
|
||||
import asyncio
|
||||
import logging
|
||||
import warnings
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Annotated, List, Literal, Dict, Optional, Any
|
||||
from datetime import datetime, timedelta, timezone, UTC
|
||||
from typing import Annotated, Literal, Optional, Any
|
||||
from langchain_core.tools import BaseTool, StructuredTool, tool, ToolException, InjectedToolArg
|
||||
from langchain_core.messages import HumanMessage, AIMessage, MessageLikeRepresentation, filter_messages
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
@ -28,7 +28,7 @@ TAVILY_SEARCH_DESCRIPTION = (
|
||||
)
|
||||
@tool(description=TAVILY_SEARCH_DESCRIPTION)
|
||||
async def tavily_search(
|
||||
queries: List[str],
|
||||
queries: list[str],
|
||||
max_results: Annotated[int, InjectedToolArg] = 5,
|
||||
topic: Annotated[Literal["general", "news", "finance"], InjectedToolArg] = "general",
|
||||
config: RunnableConfig = None
|
||||
@ -52,7 +52,7 @@ async def tavily_search(
|
||||
config=config
|
||||
)
|
||||
# Format the search results and deduplicate results by URL
|
||||
formatted_output = f"Search results: \n\n"
|
||||
formatted_output = "Search results: \n\n"
|
||||
unique_results = {}
|
||||
for response in search_results:
|
||||
for result in response['results']:
|
||||
@ -116,7 +116,7 @@ async def summarize_webpage(model: BaseChatModel, webpage_content: str) -> str:
|
||||
timeout=60.0
|
||||
)
|
||||
return f"""<summary>\n{summary.summary}\n</summary>\n\n<key_excerpts>\n{summary.key_excerpts}\n</key_excerpts>"""
|
||||
except (asyncio.TimeoutError, Exception) as e:
|
||||
except (TimeoutError, Exception) as e:
|
||||
print(f"Failed to summarize webpage: {str(e)}")
|
||||
return webpage_content
|
||||
|
||||
@ -127,7 +127,7 @@ async def summarize_webpage(model: BaseChatModel, webpage_content: str) -> str:
|
||||
async def get_mcp_access_token(
|
||||
supabase_token: str,
|
||||
base_mcp_url: str,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
) -> dict[str, Any] | None:
|
||||
try:
|
||||
form_data = {
|
||||
"client_id": "mcp_default",
|
||||
@ -165,7 +165,7 @@ async def get_tokens(config: RunnableConfig):
|
||||
return None
|
||||
expires_in = tokens.value.get("expires_in") # seconds until expiration
|
||||
created_at = tokens.created_at # datetime of token creation
|
||||
current_time = datetime.now(timezone.utc)
|
||||
current_time = datetime.now(UTC)
|
||||
expiration_time = created_at + timedelta(seconds=expires_in)
|
||||
if current_time > expiration_time:
|
||||
await store.adelete((user_id, "tokens"), "data")
|
||||
@ -260,7 +260,7 @@ async def load_mcp_tools(
|
||||
except Exception as e:
|
||||
print(f"Error loading MCP tools: {e}")
|
||||
return []
|
||||
for tool in mcp_tools:
|
||||
for tool in mcp_tools: # noqa: F402
|
||||
if tool.name in existing_tool_names:
|
||||
warnings.warn(
|
||||
f"Trying to add MCP tool with a name {tool.name} that is already in use - this tool will be ignored."
|
||||
@ -492,4 +492,4 @@ def get_tavily_api_key(config: RunnableConfig):
|
||||
return None
|
||||
return api_keys.get("TAVILY_API_KEY")
|
||||
else:
|
||||
return os.getenv("TAVILY_API_KEY")
|
||||
return os.getenv("TAVILY_API_KEY")
|
||||
|
||||
@ -6,4 +6,4 @@ class ReActAgent(BaseAgent):
|
||||
|
||||
async def get_graph(self, **kwargs):
|
||||
from .workflows import graph
|
||||
return graph
|
||||
return graph
|
||||
|
||||
@ -182,9 +182,9 @@ class BaseModelConfiguration(BaseModel):
|
||||
@classmethod
|
||||
def from_runnable_config(
|
||||
cls,
|
||||
config: Optional["RunnableConfig"] = None,
|
||||
module_name: Optional[str] = None,
|
||||
) -> "BaseModelConfiguration":
|
||||
config: RunnableConfig | None = None,
|
||||
module_name: str | None = None,
|
||||
) -> BaseModelConfiguration:
|
||||
"""
|
||||
从 RunnableConfig 和 YAML 文件中构建 Configuration 实例
|
||||
"""
|
||||
@ -246,29 +246,30 @@ class BaseModelConfiguration(BaseModel):
|
||||
confs: dict[str, Any] = {}
|
||||
configurable_items: dict[str, Any] = {}
|
||||
|
||||
for name, field in cls.model_fields.items():
|
||||
for name, _field in cls.model_fields.items():
|
||||
value = getattr(instance, name)
|
||||
|
||||
confs[name] = value
|
||||
|
||||
# 安全地处理 Pydantic 字段元数据
|
||||
field_metadata = {}
|
||||
if hasattr(field, 'json_schema_extra') and field.json_schema_extra:
|
||||
if isinstance(field.json_schema_extra, dict):
|
||||
field_metadata = field.json_schema_extra['metadata']
|
||||
elif isinstance(field.json_schema_extra, (list, tuple)):
|
||||
if hasattr(_field, 'json_schema_extra') and _field.json_schema_extra:
|
||||
if isinstance(_field.json_schema_extra, dict):
|
||||
field_metadata = _field.json_schema_extra['metadata']
|
||||
elif isinstance(_field.json_schema_extra, list | tuple):
|
||||
# 在 Pydantic v2 中,metadata 可能是列表,合并所有字典项
|
||||
for item in field.json_schema_extra:
|
||||
for item in _field.json_schema_extra:
|
||||
if isinstance(item, dict):
|
||||
field_metadata.update(item['metadata'])
|
||||
|
||||
# 检查字段是否应该可配置 - 支持不同的元数据格式
|
||||
if field_metadata.get("configurable", True):
|
||||
default_type = _field.annotation.__name__ if hasattr(_field.annotation, '__name__') else 'str'
|
||||
configurable_items[name] = {
|
||||
"type": field_metadata.get("type", field.annotation.__name__),
|
||||
"type": field_metadata.get("type", default_type),
|
||||
"name": field_metadata.get("name") or name,
|
||||
"options": field_metadata.get("options") or [],
|
||||
"default": field.default if field.default is not None else None,
|
||||
"default": _field.default if _field.default is not None else None,
|
||||
"description": field_metadata.get("description") or "",
|
||||
"x_oap_ui_config": field_metadata.get("x_oap_ui_config", {}),
|
||||
}
|
||||
|
||||
@ -3,7 +3,7 @@ import time
|
||||
import traceback
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, List, Any
|
||||
from typing import Optional, Any
|
||||
from datetime import datetime
|
||||
|
||||
import chromadb
|
||||
@ -11,10 +11,8 @@ from chromadb.config import Settings
|
||||
from chromadb.api.types import EmbeddingFunction, Documents, Embeddings
|
||||
from chromadb.utils.embedding_functions import OpenAIEmbeddingFunction
|
||||
|
||||
|
||||
|
||||
from src.knowledge.knowledge_base import KnowledgeBase
|
||||
from src.knowledge.kb_utils import split_text_into_chunks, prepare_item_metadata, get_embedding_config
|
||||
from src.knowledge.kb_utils import split_text_into_chunks, split_text_into_qa_chunks, prepare_item_metadata, get_embedding_config
|
||||
from src.utils import logger, hashstr
|
||||
from src import config
|
||||
|
||||
@ -46,12 +44,7 @@ class ChromaKB(KnowledgeBase):
|
||||
)
|
||||
|
||||
# 存储集合映射 {db_id: collection}
|
||||
self.collections: Dict[str, Any] = {}
|
||||
|
||||
# 分块配置
|
||||
self.chunk_size = kwargs.get('chunk_size', 1000)
|
||||
self.chunk_overlap = kwargs.get('chunk_overlap', 200)
|
||||
|
||||
self.collections: dict[str, Any] = {}
|
||||
logger.info("ChromaKB initialized")
|
||||
|
||||
@property
|
||||
@ -59,7 +52,7 @@ class ChromaKB(KnowledgeBase):
|
||||
"""知识库类型标识"""
|
||||
return "chroma"
|
||||
|
||||
async def _create_kb_instance(self, db_id: str, config: Dict) -> Any:
|
||||
async def _create_kb_instance(self, db_id: str, kb_config: dict) -> Any:
|
||||
"""创建向量数据库集合"""
|
||||
logger.info(f"Creating ChromaDB collection for {db_id}")
|
||||
|
||||
@ -92,7 +85,7 @@ class ChromaKB(KnowledgeBase):
|
||||
self.chroma_client.delete_collection(name=collection_name)
|
||||
raise Exception("Model mismatch, recreating collection")
|
||||
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
# 创建新集合
|
||||
logger.info(f"Creating new collection with embedding model: {embed_info.get('name', 'default')}")
|
||||
collection_metadata = {
|
||||
@ -113,7 +106,7 @@ class ChromaKB(KnowledgeBase):
|
||||
"""初始化向量数据库集合(无需特殊初始化)"""
|
||||
pass
|
||||
|
||||
def _get_embedding_function(self, embed_info: Dict):
|
||||
def _get_embedding_function(self, embed_info: dict):
|
||||
"""获取 embedding 函数"""
|
||||
config_dict = get_embedding_config(embed_info)
|
||||
|
||||
@ -144,22 +137,32 @@ class ChromaKB(KnowledgeBase):
|
||||
logger.error(f"Traceback: {traceback.format_exc()}")
|
||||
return None
|
||||
|
||||
def _split_text_into_chunks(self, text: str, file_id: str, filename: str) -> List[Dict]:
|
||||
def _split_text_into_chunks(self, text: str, file_id: str, filename: str, params: dict) -> list[dict]:
|
||||
"""将文本分割成块"""
|
||||
chunks = split_text_into_chunks(text, file_id, filename, self.chunk_size, self.chunk_overlap)
|
||||
# 检查是否使用QA分割模式
|
||||
use_qa_split = params.get('use_qa_split', False)
|
||||
|
||||
if use_qa_split:
|
||||
# 使用QA分割模式
|
||||
qa_separator = params.get('qa_separator', '\n\n\n')
|
||||
chunks = split_text_into_qa_chunks(text, file_id, filename, qa_separator, params)
|
||||
else:
|
||||
# 使用传统分割模式
|
||||
chunks = split_text_into_chunks(text, file_id, filename, params)
|
||||
|
||||
# 为 ChromaDB 添加特定的 metadata 格式
|
||||
for chunk in chunks:
|
||||
chunk["metadata"] = {
|
||||
"source": chunk["source"],
|
||||
"chunk_id": chunk["chunk_id"],
|
||||
"full_doc_id": file_id
|
||||
"full_doc_id": file_id,
|
||||
"chunk_type": chunk.get("chunk_type", "normal") # 添加chunk类型标识
|
||||
}
|
||||
|
||||
return chunks
|
||||
|
||||
async def add_content(self, db_id: str, items: List[str],
|
||||
params: Optional[Dict] = None) -> List[Dict]:
|
||||
async def add_content(self, db_id: str, items: list[str],
|
||||
params:dict | None) -> list[dict]:
|
||||
"""添加内容(文件/URL)"""
|
||||
if db_id not in self.databases_meta:
|
||||
raise ValueError(f"Database {db_id} not found")
|
||||
@ -176,7 +179,6 @@ class ChromaKB(KnowledgeBase):
|
||||
metadata = prepare_item_metadata(item, content_type, db_id)
|
||||
file_id = metadata["file_id"]
|
||||
filename = metadata["filename"]
|
||||
item_path = metadata["path"]
|
||||
|
||||
# 添加文件记录
|
||||
file_record = metadata.copy()
|
||||
@ -191,7 +193,7 @@ class ChromaKB(KnowledgeBase):
|
||||
markdown_content = await self._process_url_to_markdown(item, params=params)
|
||||
|
||||
# 分割文本成块
|
||||
chunks = self._split_text_into_chunks(markdown_content, file_id, filename)
|
||||
chunks = self._split_text_into_chunks(markdown_content, file_id, filename, params)
|
||||
logger.info(f"Split {filename} into {len(chunks)} chunks")
|
||||
|
||||
# 准备向量数据库插入的数据
|
||||
@ -302,7 +304,7 @@ class ChromaKB(KnowledgeBase):
|
||||
del self.files_meta[file_id]
|
||||
self._save_metadata()
|
||||
|
||||
async def get_file_info(self, db_id: str, file_id: str) -> Dict:
|
||||
async def get_file_info(self, db_id: str, file_id: str) -> dict:
|
||||
"""获取文件信息和chunks"""
|
||||
if file_id not in self.files_meta:
|
||||
raise Exception(f"File not found: {file_id}")
|
||||
@ -336,4 +338,4 @@ class ChromaKB(KnowledgeBase):
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting chunks for file {file_id}: {e}")
|
||||
|
||||
return {"lines": []}
|
||||
return {"lines": []}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
from typing import Dict, Type, Any
|
||||
from typing import Any
|
||||
from src.knowledge.knowledge_base import KnowledgeBase, KBNotFoundError
|
||||
from src.utils import logger
|
||||
|
||||
@ -7,14 +7,14 @@ class KnowledgeBaseFactory:
|
||||
"""知识库工厂类,负责创建不同类型的知识库实例"""
|
||||
|
||||
# 注册的知识库类型映射 {kb_type: kb_class}
|
||||
_kb_types: Dict[str, Type[KnowledgeBase]] = {}
|
||||
_kb_types: dict[str, type[KnowledgeBase]] = {}
|
||||
|
||||
# 每种类型的默认配置
|
||||
_default_configs: Dict[str, Dict] = {}
|
||||
_default_configs: dict[str, dict] = {}
|
||||
|
||||
@classmethod
|
||||
def register(cls, kb_type: str, kb_class: Type[KnowledgeBase],
|
||||
default_config: Dict = None):
|
||||
def register(cls, kb_type: str, kb_class: type[KnowledgeBase],
|
||||
default_config: dict = None):
|
||||
"""
|
||||
注册知识库类型
|
||||
|
||||
@ -24,7 +24,7 @@ class KnowledgeBaseFactory:
|
||||
default_config: 默认配置
|
||||
"""
|
||||
if not issubclass(kb_class, KnowledgeBase):
|
||||
raise ValueError(f"Knowledge base class must inherit from KnowledgeBase")
|
||||
raise ValueError("Knowledge base class must inherit from KnowledgeBase")
|
||||
|
||||
cls._kb_types[kb_type] = kb_class
|
||||
cls._default_configs[kb_type] = default_config or {}
|
||||
@ -67,7 +67,7 @@ class KnowledgeBaseFactory:
|
||||
raise
|
||||
|
||||
@classmethod
|
||||
def get_available_types(cls) -> Dict[str, Dict]:
|
||||
def get_available_types(cls) -> dict[str, dict]:
|
||||
"""
|
||||
获取所有可用的知识库类型
|
||||
|
||||
@ -97,7 +97,7 @@ class KnowledgeBaseFactory:
|
||||
return kb_type in cls._kb_types
|
||||
|
||||
@classmethod
|
||||
def get_default_config(cls, kb_type: str) -> Dict:
|
||||
def get_default_config(cls, kb_type: str) -> dict:
|
||||
"""
|
||||
获取指定类型的默认配置
|
||||
|
||||
@ -107,4 +107,4 @@ class KnowledgeBaseFactory:
|
||||
Returns:
|
||||
默认配置字典
|
||||
"""
|
||||
return cls._default_configs.get(kb_type, {}).copy()
|
||||
return cls._default_configs.get(kb_type, {}).copy()
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
from typing import Dict, Optional, List, Any
|
||||
from typing import Any
|
||||
from datetime import datetime
|
||||
|
||||
from src.knowledge.knowledge_base import KnowledgeBase, KBNotFoundError, KBOperationError
|
||||
@ -27,10 +27,10 @@ class KnowledgeBaseManager:
|
||||
os.makedirs(work_dir, exist_ok=True)
|
||||
|
||||
# 知识库实例缓存 {kb_type: kb_instance}
|
||||
self.kb_instances: Dict[str, KnowledgeBase] = {}
|
||||
self.kb_instances: dict[str, KnowledgeBase] = {}
|
||||
|
||||
# 全局数据库元信息 {db_id: metadata_with_kb_type}
|
||||
self.global_databases_meta: Dict[str, Dict] = {}
|
||||
self.global_databases_meta: dict[str, dict] = {}
|
||||
|
||||
# 加载全局元数据
|
||||
self._load_global_metadata()
|
||||
@ -128,7 +128,7 @@ class KnowledgeBaseManager:
|
||||
# 统一的外部接口 - 与原始 LightRagBasedKB 兼容
|
||||
# =============================================================================
|
||||
|
||||
def get_databases(self) -> Dict:
|
||||
def get_databases(self) -> dict:
|
||||
"""获取所有数据库信息"""
|
||||
all_databases = []
|
||||
|
||||
@ -139,9 +139,7 @@ class KnowledgeBaseManager:
|
||||
|
||||
return {"databases": all_databases}
|
||||
|
||||
def create_database(self, database_name: str, description: str,
|
||||
kb_type: str = "lightrag", embed_info: Optional[Dict] = None,
|
||||
**kwargs) -> Dict:
|
||||
def create_database(self, database_name: str, description: str, kb_type: str, embed_info: dict | None = None, **kwargs) -> dict:
|
||||
"""
|
||||
创建数据库
|
||||
|
||||
@ -150,7 +148,7 @@ class KnowledgeBaseManager:
|
||||
description: 数据库描述
|
||||
kb_type: 知识库类型,默认为lightrag
|
||||
embed_info: 嵌入模型信息
|
||||
**kwargs: 其他配置参数
|
||||
**kwargs: 其他配置参数,包括chunk_size和chunk_overlap
|
||||
|
||||
Returns:
|
||||
数据库信息字典
|
||||
@ -158,8 +156,7 @@ class KnowledgeBaseManager:
|
||||
# 验证知识库类型
|
||||
if not KnowledgeBaseFactory.is_type_supported(kb_type):
|
||||
available_types = list(KnowledgeBaseFactory.get_available_types().keys())
|
||||
raise ValueError(f"Unsupported knowledge base type: {kb_type}. "
|
||||
f"Available types: {available_types}")
|
||||
raise ValueError(f"Unsupported knowledge base type: {kb_type}. Available types: {available_types}")
|
||||
|
||||
# 获取或创建对应类型的知识库实例
|
||||
kb_instance = self._get_or_create_kb_instance(kb_type)
|
||||
@ -174,14 +171,16 @@ class KnowledgeBaseManager:
|
||||
"name": database_name,
|
||||
"description": description,
|
||||
"kb_type": kb_type,
|
||||
"created_at": datetime.now().isoformat()
|
||||
"created_at": datetime.now().isoformat(),
|
||||
"additional_params": kwargs.copy() # 将所有额外参数存储在additional_params中
|
||||
}
|
||||
|
||||
self._save_global_metadata()
|
||||
|
||||
logger.info(f"Created {kb_type} database: {database_name} ({db_id})")
|
||||
logger.info(f"Created {kb_type} database: {database_name} ({db_id}) with {kwargs}")
|
||||
return db_info
|
||||
|
||||
def delete_database(self, db_id: str) -> Dict:
|
||||
def delete_database(self, db_id: str) -> dict:
|
||||
"""删除数据库"""
|
||||
try:
|
||||
kb_instance = self._get_kb_for_database(db_id)
|
||||
@ -197,11 +196,10 @@ class KnowledgeBaseManager:
|
||||
logger.warning(f"Database {db_id} not found during deletion: {e}")
|
||||
return {"message": "删除成功"} # 兼容性:即使不存在也返回成功
|
||||
|
||||
async def add_content(self, db_id: str, items: List[str],
|
||||
params: Optional[Dict] = None) -> List[Dict]:
|
||||
async def add_content(self, db_id: str, items: list[str], params: dict | None = None) -> list[dict]:
|
||||
"""添加内容(文件/URL)"""
|
||||
kb_instance = self._get_kb_for_database(db_id)
|
||||
return await kb_instance.add_content(db_id, items, params)
|
||||
return await kb_instance.add_content(db_id, items, params or {})
|
||||
|
||||
async def aquery(self, query_text: str, db_id: str, **kwargs) -> str:
|
||||
"""异步查询知识库"""
|
||||
@ -213,11 +211,20 @@ class KnowledgeBaseManager:
|
||||
kb_instance = self._get_kb_for_database(db_id)
|
||||
return kb_instance.query(query_text, db_id, **kwargs)
|
||||
|
||||
def get_database_info(self, db_id: str) -> Optional[Dict]:
|
||||
def get_database_info(self, db_id: str) -> dict | None:
|
||||
"""获取数据库详细信息"""
|
||||
try:
|
||||
kb_instance = self._get_kb_for_database(db_id)
|
||||
return kb_instance.get_database_info(db_id)
|
||||
db_info = kb_instance.get_database_info(db_id)
|
||||
|
||||
# 添加全局元数据中的additional_params信息
|
||||
if db_info and db_id in self.global_databases_meta:
|
||||
global_meta = self.global_databases_meta[db_id]
|
||||
additional_params = global_meta.get("additional_params", {})
|
||||
if additional_params:
|
||||
db_info["additional_params"] = additional_params
|
||||
|
||||
return db_info
|
||||
except KBNotFoundError:
|
||||
return None
|
||||
|
||||
@ -226,12 +233,12 @@ class KnowledgeBaseManager:
|
||||
kb_instance = self._get_kb_for_database(db_id)
|
||||
await kb_instance.delete_file(db_id, file_id)
|
||||
|
||||
async def get_file_info(self, db_id: str, file_id: str) -> Dict:
|
||||
async def get_file_info(self, db_id: str, file_id: str) -> dict:
|
||||
"""获取文件信息"""
|
||||
kb_instance = self._get_kb_for_database(db_id)
|
||||
return await kb_instance.get_file_info(db_id, file_id)
|
||||
|
||||
def get_db_upload_path(self, db_id: Optional[str] = None) -> str:
|
||||
def get_db_upload_path(self, db_id: str | None = None) -> str:
|
||||
"""获取数据库上传路径"""
|
||||
if db_id:
|
||||
try:
|
||||
@ -246,7 +253,7 @@ class KnowledgeBaseManager:
|
||||
os.makedirs(general_uploads, exist_ok=True)
|
||||
return general_uploads
|
||||
|
||||
def update_database(self, db_id: str, name: str, description: str) -> Dict:
|
||||
def update_database(self, db_id: str, name: str, description: str) -> dict:
|
||||
"""更新数据库"""
|
||||
kb_instance = self._get_kb_for_database(db_id)
|
||||
result = kb_instance.update_database(db_id, name, description)
|
||||
@ -259,7 +266,7 @@ class KnowledgeBaseManager:
|
||||
|
||||
return result
|
||||
|
||||
def get_retrievers(self) -> Dict[str, Dict]:
|
||||
def get_retrievers(self) -> dict[str, dict]:
|
||||
"""获取所有检索器"""
|
||||
all_retrievers = {}
|
||||
|
||||
@ -274,11 +281,11 @@ class KnowledgeBaseManager:
|
||||
# 管理器特有的方法
|
||||
# =============================================================================
|
||||
|
||||
def get_supported_kb_types(self) -> Dict[str, Dict]:
|
||||
def get_supported_kb_types(self) -> dict[str, dict]:
|
||||
"""获取支持的知识库类型"""
|
||||
return KnowledgeBaseFactory.get_available_types()
|
||||
|
||||
def get_kb_instance_info(self) -> Dict[str, Dict]:
|
||||
def get_kb_instance_info(self) -> dict[str, dict]:
|
||||
"""获取知识库实例信息"""
|
||||
info = {}
|
||||
for kb_type, kb_instance in self.kb_instances.items():
|
||||
@ -289,7 +296,7 @@ class KnowledgeBaseManager:
|
||||
}
|
||||
return info
|
||||
|
||||
def migrate_database(self, db_id: str, target_kb_type: str) -> Dict:
|
||||
def migrate_database(self, db_id: str, target_kb_type: str) -> dict:
|
||||
"""
|
||||
迁移数据库到不同的知识库类型
|
||||
|
||||
@ -305,7 +312,7 @@ class KnowledgeBaseManager:
|
||||
# TODO: 实现数据库迁移逻辑
|
||||
raise NotImplementedError("Database migration not implemented yet")
|
||||
|
||||
def get_statistics(self) -> Dict:
|
||||
def get_statistics(self) -> dict:
|
||||
"""获取统计信息"""
|
||||
stats = {
|
||||
"total_databases": len(self.global_databases_meta),
|
||||
@ -386,7 +393,7 @@ class KnowledgeBaseManager:
|
||||
kb_type = self.global_databases_meta[db_id].get("kb_type", "lightrag")
|
||||
return kb_type == "lightrag"
|
||||
|
||||
def get_lightrag_databases(self) -> List[Dict]:
|
||||
def get_lightrag_databases(self) -> list[dict]:
|
||||
"""
|
||||
获取所有 LightRAG 类型的数据库
|
||||
|
||||
@ -400,4 +407,4 @@ class KnowledgeBaseManager:
|
||||
if db.get("kb_type", "lightrag") == "lightrag":
|
||||
lightrag_databases.append(db)
|
||||
|
||||
return lightrag_databases
|
||||
return lightrag_databases
|
||||
|
||||
@ -1,44 +1,35 @@
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any
|
||||
from typing import Any
|
||||
from langchain_text_splitters import MarkdownTextSplitter
|
||||
from src.utils import hashstr, get_docker_safe_url, logger
|
||||
from src import config
|
||||
|
||||
|
||||
def split_text_into_chunks(text: str, file_id: str, filename: str,
|
||||
chunk_size: int = 1000, chunk_overlap: int = 200) -> List[Dict]:
|
||||
def split_text_into_chunks(text: str, file_id: str, filename: str, params: dict = {}) -> list[dict]:
|
||||
"""
|
||||
将文本分割成块
|
||||
|
||||
Args:
|
||||
text: 要分割的文本
|
||||
file_id: 文件ID
|
||||
filename: 文件名
|
||||
chunk_size: 块大小
|
||||
chunk_overlap: 块重叠大小
|
||||
|
||||
Returns:
|
||||
List[Dict]: 分割后的文本块列表
|
||||
将文本分割成块,使用 LangChain 的 MarkdownTextSplitter 进行智能分割
|
||||
"""
|
||||
chunks = []
|
||||
chunk_size = params.get('chunk_size', 1000)
|
||||
chunk_overlap = params.get('chunk_overlap', 200)
|
||||
|
||||
# 简单的分块策略:按段落和长度分割
|
||||
paragraphs = text.split('\n\n')
|
||||
# 使用 MarkdownTextSplitter 进行智能分割
|
||||
# MarkdownTextSplitter 会尝试沿着 Markdown 格式的标题进行分割
|
||||
text_splitter = MarkdownTextSplitter(
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=chunk_overlap,
|
||||
)
|
||||
|
||||
current_chunk = ""
|
||||
chunk_index = 0
|
||||
text_chunks = text_splitter.split_text(text)
|
||||
|
||||
for paragraph in paragraphs:
|
||||
paragraph = paragraph.strip()
|
||||
if not paragraph:
|
||||
continue
|
||||
|
||||
# 如果当前块加上新段落会超过限制,保存当前块
|
||||
if len(current_chunk) + len(paragraph) > chunk_size and current_chunk:
|
||||
# 转换为标准格式
|
||||
for chunk_index, chunk_content in enumerate(text_chunks):
|
||||
if chunk_content.strip(): # 跳过空块
|
||||
chunks.append({
|
||||
"id": f"{file_id}_chunk_{chunk_index}",
|
||||
"content": current_chunk.strip(),
|
||||
"content": chunk_content.strip(),
|
||||
"file_id": file_id,
|
||||
"filename": filename,
|
||||
"chunk_index": chunk_index,
|
||||
@ -46,44 +37,13 @@ def split_text_into_chunks(text: str, file_id: str, filename: str,
|
||||
"chunk_id": f"{file_id}_chunk_{chunk_index}"
|
||||
})
|
||||
|
||||
# 开始新块,包含重叠内容
|
||||
if len(current_chunk) > chunk_overlap:
|
||||
current_chunk = current_chunk[-chunk_overlap:] + "\n\n" + paragraph
|
||||
else:
|
||||
current_chunk = paragraph
|
||||
chunk_index += 1
|
||||
else:
|
||||
if current_chunk:
|
||||
current_chunk += "\n\n" + paragraph
|
||||
else:
|
||||
current_chunk = paragraph
|
||||
|
||||
# 添加最后一块
|
||||
if current_chunk.strip():
|
||||
chunks.append({
|
||||
"id": f"{file_id}_chunk_{chunk_index}",
|
||||
"content": current_chunk.strip(),
|
||||
"file_id": file_id,
|
||||
"filename": filename,
|
||||
"chunk_index": chunk_index,
|
||||
"source": filename,
|
||||
"chunk_id": f"{file_id}_chunk_{chunk_index}"
|
||||
})
|
||||
|
||||
logger.debug(f"Successfully split text into {len(chunks)} chunks using MarkdownTextSplitter")
|
||||
return chunks
|
||||
|
||||
|
||||
def prepare_item_metadata(item: str, content_type: str, db_id: str) -> Dict:
|
||||
def prepare_item_metadata(item: str, content_type: str, db_id: str) -> dict:
|
||||
"""
|
||||
准备文件或URL的元数据
|
||||
|
||||
Args:
|
||||
item: 文件路径或URL
|
||||
content_type: 内容类型 ('file' 或 'url')
|
||||
db_id: 数据库ID
|
||||
|
||||
Returns:
|
||||
Dict: 包含元数据的字典
|
||||
"""
|
||||
if content_type == "file":
|
||||
file_path = Path(item)
|
||||
@ -108,7 +68,35 @@ def prepare_item_metadata(item: str, content_type: str, db_id: str) -> Dict:
|
||||
}
|
||||
|
||||
|
||||
def get_embedding_config(embed_info: Dict) -> Dict:
|
||||
def split_text_into_qa_chunks(text: str, file_id: str, filename: str,
|
||||
qa_separator: None | str = None, params: dict = {}) -> list[dict]:
|
||||
"""
|
||||
将文本按QA对分割成块,使用 LangChain 的 CharacterTextSplitter 进行分割"""
|
||||
qa_separator = qa_separator or '\n\n'
|
||||
text_chunks = text.split(qa_separator)
|
||||
|
||||
# 转换为标准格式
|
||||
chunks = []
|
||||
for chunk_index, chunk_content in enumerate(text_chunks):
|
||||
if chunk_content.strip(): # 跳过空块
|
||||
chunk_content = chunk_content.strip()[:4096]
|
||||
chunks.append({
|
||||
"id": f"{file_id}_qa_chunk_{chunk_index}",
|
||||
"content": chunk_content.strip(),
|
||||
"file_id": file_id,
|
||||
"filename": filename,
|
||||
"chunk_index": chunk_index,
|
||||
"source": filename,
|
||||
"chunk_id": f"{file_id}_qa_chunk_{chunk_index}",
|
||||
"chunk_type": "qa" # 标识为QA类型的chunk
|
||||
})
|
||||
|
||||
logger.debug(f"QA chunks: {chunks[0]}")
|
||||
logger.debug(f"Successfully split QA text into {len(chunks)} chunks using CharacterTextSplitter with `{qa_separator=}`")
|
||||
return chunks
|
||||
|
||||
|
||||
def get_embedding_config(embed_info: dict) -> dict:
|
||||
"""
|
||||
获取嵌入模型配置
|
||||
|
||||
@ -116,7 +104,7 @@ def get_embedding_config(embed_info: Dict) -> Dict:
|
||||
embed_info: 嵌入信息字典
|
||||
|
||||
Returns:
|
||||
Dict: 标准化的嵌入配置
|
||||
dict: 标准化的嵌入配置
|
||||
"""
|
||||
config_dict = {}
|
||||
|
||||
@ -134,4 +122,4 @@ def get_embedding_config(embed_info: Dict) -> Dict:
|
||||
config_dict['dimension'] = getattr(default_model, 'dimension', 1024)
|
||||
|
||||
logger.debug(f"Embedding config: {config_dict}")
|
||||
return config_dict
|
||||
return config_dict
|
||||
|
||||
@ -2,7 +2,8 @@ import os
|
||||
import json
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import List, Dict, Optional, Any, AsyncGenerator
|
||||
from typing import Any
|
||||
from collections.abc import AsyncGenerator
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
@ -35,8 +36,8 @@ class KnowledgeBase(ABC):
|
||||
work_dir: 工作目录
|
||||
"""
|
||||
self.work_dir = work_dir
|
||||
self.databases_meta: Dict[str, Dict] = {}
|
||||
self.files_meta: Dict[str, Dict] = {}
|
||||
self.databases_meta: dict[str, dict] = {}
|
||||
self.files_meta: dict[str, dict] = {}
|
||||
os.makedirs(work_dir, exist_ok=True)
|
||||
|
||||
# 自动加载元数据
|
||||
@ -49,7 +50,7 @@ class KnowledgeBase(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def _create_kb_instance(self, db_id: str, config: Dict) -> Any:
|
||||
async def _create_kb_instance(self, db_id: str, config: dict) -> Any:
|
||||
"""
|
||||
创建底层知识库实例
|
||||
|
||||
@ -73,7 +74,7 @@ class KnowledgeBase(ABC):
|
||||
pass
|
||||
|
||||
def create_database(self, database_name: str, description: str,
|
||||
embed_info: Optional[Dict] = None, **kwargs) -> Dict:
|
||||
embed_info: dict | None = None, **kwargs) -> dict:
|
||||
"""
|
||||
创建数据库
|
||||
|
||||
@ -112,7 +113,7 @@ class KnowledgeBase(ABC):
|
||||
|
||||
return db_dict
|
||||
|
||||
def delete_database(self, db_id: str) -> Dict:
|
||||
def delete_database(self, db_id: str) -> dict:
|
||||
"""
|
||||
删除数据库
|
||||
|
||||
@ -145,8 +146,8 @@ class KnowledgeBase(ABC):
|
||||
return {"message": "删除成功"}
|
||||
|
||||
@abstractmethod
|
||||
async def add_content(self, db_id: str, items: List[str],
|
||||
params: Optional[Dict] = None) -> List[Dict]:
|
||||
async def add_content(self, db_id: str, items: list[str],
|
||||
params: dict | None = None) -> list[dict]:
|
||||
"""
|
||||
添加内容(文件/URL)
|
||||
|
||||
@ -191,7 +192,7 @@ class KnowledgeBase(ABC):
|
||||
logger.warning("query is deprecated, use aquery instead")
|
||||
return asyncio.run(self.aquery(query_text, db_id, **kwargs))
|
||||
|
||||
def get_database_info(self, db_id: str) -> Optional[Dict]:
|
||||
def get_database_info(self, db_id: str) -> dict | None:
|
||||
"""
|
||||
获取数据库详细信息
|
||||
|
||||
@ -225,7 +226,7 @@ class KnowledgeBase(ABC):
|
||||
meta["status"] = "已连接"
|
||||
return meta
|
||||
|
||||
def get_databases(self) -> Dict:
|
||||
def get_databases(self) -> dict:
|
||||
"""
|
||||
获取所有数据库信息
|
||||
|
||||
@ -269,7 +270,7 @@ class KnowledgeBase(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def get_file_info(self, db_id: str, file_id: str) -> Dict:
|
||||
async def get_file_info(self, db_id: str, file_id: str) -> dict:
|
||||
"""
|
||||
获取文件信息和chunks
|
||||
|
||||
@ -282,7 +283,7 @@ class KnowledgeBase(ABC):
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_db_upload_path(self, db_id: Optional[str] = None) -> str:
|
||||
def get_db_upload_path(self, db_id: str | None = None) -> str:
|
||||
"""
|
||||
获取数据库上传路径
|
||||
|
||||
@ -301,7 +302,7 @@ class KnowledgeBase(ABC):
|
||||
os.makedirs(general_uploads, exist_ok=True)
|
||||
return general_uploads
|
||||
|
||||
def update_database(self, db_id: str, name: str, description: str) -> Dict:
|
||||
def update_database(self, db_id: str, name: str, description: str) -> dict:
|
||||
"""
|
||||
更新数据库
|
||||
|
||||
@ -322,7 +323,7 @@ class KnowledgeBase(ABC):
|
||||
|
||||
return self.get_database_info(db_id)
|
||||
|
||||
def get_retrievers(self) -> Dict[str, Dict]:
|
||||
def get_retrievers(self) -> dict[str, dict]:
|
||||
"""
|
||||
获取所有检索器
|
||||
|
||||
@ -371,8 +372,7 @@ class KnowledgeBase(ABC):
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save {self.kb_type} metadata: {e}")
|
||||
|
||||
async def _process_file_to_markdown(self, file_path: str,
|
||||
params: Optional[Dict] = None) -> str:
|
||||
async def _process_file_to_markdown(self, file_path: str, params: dict | None = None) -> str:
|
||||
"""
|
||||
将不同类型的文件转换为markdown格式
|
||||
|
||||
@ -411,18 +411,20 @@ class KnowledgeBase(ABC):
|
||||
text = ocr.process_image(str(file_path_obj))
|
||||
return f"# {file_path_obj.name}\n\n{text}"
|
||||
|
||||
elif file_ext in ['.html', '.htm']:
|
||||
# 使用 BeautifulSoup 处理 HTML 文件
|
||||
from markdownify import markdownify as md
|
||||
with open(file_path_obj, encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
text = md(content, heading_style="ATX")
|
||||
return f"# {file_path_obj.name}\n\n{text}"
|
||||
|
||||
else:
|
||||
# 尝试作为文本文件读取
|
||||
try:
|
||||
import textract # type: ignore
|
||||
text = textract.process(file_path_obj).decode('utf-8')
|
||||
return f"# {file_path_obj.name}\n\n{text}"
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to process file {file_path_obj}: {e}")
|
||||
return f"# {file_path_obj.name}\n\nFailed to process file: {e}"
|
||||
raise ValueError(f"Unsupported file type: {file_ext}")
|
||||
|
||||
async def _process_url_to_markdown(self, url: str,
|
||||
params: Optional[Dict] = None) -> str:
|
||||
params: dict | None = None) -> str:
|
||||
"""
|
||||
将URL转换为markdown格式
|
||||
|
||||
@ -443,4 +445,4 @@ class KnowledgeBase(ABC):
|
||||
return f"# {url}\n\n{text_content}"
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to process URL {url}: {e}")
|
||||
return f"# {url}\n\nFailed to process URL: {e}"
|
||||
return f"# {url}\n\nFailed to process URL: {e}"
|
||||
|
||||
@ -2,7 +2,6 @@ import os
|
||||
import time
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, List, Any
|
||||
from datetime import datetime
|
||||
|
||||
from lightrag import LightRAG, QueryParam
|
||||
@ -11,7 +10,7 @@ from lightrag.utils import EmbeddingFunc, setup_logger
|
||||
from lightrag.kg.shared_storage import initialize_pipeline_status
|
||||
|
||||
from src.knowledge.knowledge_base import KnowledgeBase
|
||||
from src.knowledge.kb_utils import split_text_into_chunks, prepare_item_metadata, get_embedding_config
|
||||
from src.knowledge.kb_utils import prepare_item_metadata, get_embedding_config
|
||||
from src import config
|
||||
from src.utils import logger, hashstr, get_docker_safe_url
|
||||
|
||||
@ -34,7 +33,7 @@ class LightRagKB(KnowledgeBase):
|
||||
super().__init__(work_dir)
|
||||
|
||||
# 存储 LightRAG 实例映射 {db_id: LightRAG}
|
||||
self.instances: Dict[str, LightRAG] = {}
|
||||
self.instances: dict[str, LightRAG] = {}
|
||||
|
||||
# 设置 LightRAG 日志
|
||||
log_dir = os.path.join(work_dir, "logs", "lightrag")
|
||||
@ -49,7 +48,7 @@ class LightRagKB(KnowledgeBase):
|
||||
"""知识库类型标识"""
|
||||
return "lightrag"
|
||||
|
||||
async def _create_kb_instance(self, db_id: str, config: Dict) -> LightRAG:
|
||||
async def _create_kb_instance(self, db_id: str, kb_config: dict) -> LightRAG:
|
||||
"""创建 LightRAG 实例"""
|
||||
logger.info(f"Creating LightRAG instance for {db_id}")
|
||||
|
||||
@ -84,7 +83,7 @@ class LightRagKB(KnowledgeBase):
|
||||
await instance.initialize_storages()
|
||||
await initialize_pipeline_status()
|
||||
|
||||
async def _get_lightrag_instance(self, db_id: str) -> Optional[LightRAG]:
|
||||
async def _get_lightrag_instance(self, db_id: str) -> LightRAG | None:
|
||||
"""获取或创建 LightRAG 实例"""
|
||||
if db_id in self.instances:
|
||||
return self.instances[db_id]
|
||||
@ -107,7 +106,7 @@ class LightRagKB(KnowledgeBase):
|
||||
logger.error(f"Traceback: {traceback.format_exc()}")
|
||||
return None
|
||||
|
||||
def _get_llm_func(self, llm_info: Dict):
|
||||
def _get_llm_func(self, llm_info: dict):
|
||||
"""获取 LLM 函数"""
|
||||
from src.models import select_model
|
||||
model = select_model(LIGHTRAG_LLM_PROVIDER, LIGHTRAG_LLM_NAME)
|
||||
@ -125,7 +124,7 @@ class LightRagKB(KnowledgeBase):
|
||||
)
|
||||
return llm_model_func
|
||||
|
||||
def _get_embedding_func(self, embed_info: Dict):
|
||||
def _get_embedding_func(self, embed_info: dict):
|
||||
"""获取 embedding 函数"""
|
||||
config_dict = get_embedding_config(embed_info)
|
||||
|
||||
@ -140,8 +139,8 @@ class LightRagKB(KnowledgeBase):
|
||||
),
|
||||
)
|
||||
|
||||
async def add_content(self, db_id: str, items: List[str],
|
||||
params: Optional[Dict] = None) -> List[Dict]:
|
||||
async def add_content(self, db_id: str, items: list[str],
|
||||
params: dict | None = None) -> list[dict]:
|
||||
"""添加内容(文件/URL)"""
|
||||
if db_id not in self.databases_meta:
|
||||
raise ValueError(f"Database {db_id} not found")
|
||||
@ -157,7 +156,6 @@ class LightRagKB(KnowledgeBase):
|
||||
# 准备文件元数据
|
||||
metadata = prepare_item_metadata(item, content_type, db_id)
|
||||
file_id = metadata["file_id"]
|
||||
filename = metadata["filename"]
|
||||
item_path = metadata["path"]
|
||||
|
||||
# 添加文件记录
|
||||
@ -241,7 +239,7 @@ class LightRagKB(KnowledgeBase):
|
||||
del self.files_meta[file_id]
|
||||
self._save_metadata()
|
||||
|
||||
async def get_file_info(self, db_id: str, file_id: str) -> Dict:
|
||||
async def get_file_info(self, db_id: str, file_id: str) -> dict:
|
||||
"""获取文件信息和chunks"""
|
||||
if file_id not in self.files_meta:
|
||||
raise Exception(f"File not found: {file_id}")
|
||||
@ -269,4 +267,4 @@ class LightRagKB(KnowledgeBase):
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting chunks for file {file_id}: {e}")
|
||||
|
||||
return {"lines": []}
|
||||
return {"lines": []}
|
||||
|
||||
@ -3,25 +3,22 @@ import time
|
||||
import traceback
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, List, Any
|
||||
from typing import Any
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
|
||||
try:
|
||||
from pymilvus import (
|
||||
connections, utility, Collection, CollectionSchema,
|
||||
FieldSchema, DataType, db
|
||||
)
|
||||
MILVUS_AVAILABLE = True
|
||||
except ImportError:
|
||||
MILVUS_AVAILABLE = False
|
||||
connections = None
|
||||
utility = None
|
||||
Collection = None
|
||||
from pymilvus import (
|
||||
connections, utility, Collection, CollectionSchema,
|
||||
FieldSchema, DataType, db
|
||||
)
|
||||
|
||||
from src.knowledge.knowledge_base import KnowledgeBase
|
||||
from src.knowledge.kb_utils import split_text_into_chunks, prepare_item_metadata, get_embedding_config
|
||||
from src.utils import logger, hashstr
|
||||
from src import config
|
||||
from src.models.embedding import OtherEmbedding
|
||||
from src.knowledge.knowledge_base import KnowledgeBase
|
||||
from src.knowledge.kb_utils import split_text_into_chunks, split_text_into_qa_chunks, prepare_item_metadata, get_embedding_config
|
||||
from src.utils import logger, hashstr
|
||||
|
||||
MILVUS_AVAILABLE = True
|
||||
|
||||
|
||||
class MilvusKB(KnowledgeBase):
|
||||
@ -51,7 +48,7 @@ class MilvusKB(KnowledgeBase):
|
||||
self.connection_alias = f"milvus_{hashstr(work_dir, 6)}"
|
||||
|
||||
# 存储集合映射 {db_id: Collection}
|
||||
self.collections: Dict[str, Any] = {}
|
||||
self.collections: dict[str, Any] = {}
|
||||
|
||||
# 分块配置
|
||||
self.chunk_size = kwargs.get('chunk_size', 1000)
|
||||
@ -91,7 +88,7 @@ class MilvusKB(KnowledgeBase):
|
||||
logger.error(f"Failed to connect to Milvus: {e}")
|
||||
raise
|
||||
|
||||
async def _create_kb_instance(self, db_id: str, config: Dict) -> Any:
|
||||
async def _create_kb_instance(self, db_id: str, kb_config: dict) -> Any:
|
||||
"""创建 Milvus 集合"""
|
||||
logger.info(f"Creating Milvus collection for {db_id}")
|
||||
|
||||
@ -170,39 +167,27 @@ class MilvusKB(KnowledgeBase):
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load collection into memory: {e}")
|
||||
|
||||
def _get_embedding_function(self, embed_info: Dict):
|
||||
def _get_async_embedding_function(self, embed_info: dict):
|
||||
"""获取 embedding 函数"""
|
||||
config_dict = get_embedding_config(embed_info)
|
||||
model = config_dict["model"]
|
||||
api_key = config_dict["api_key"]
|
||||
base_url = config_dict["base_url"]
|
||||
embedding_model = OtherEmbedding(
|
||||
model=config_dict.get("model"),
|
||||
base_url=config_dict.get("base_url"),
|
||||
api_key=config_dict.get("api_key"),
|
||||
)
|
||||
|
||||
# 返回同步的嵌入函数
|
||||
def embedding_function(texts):
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
from lightrag.llm.openai import openai_embed
|
||||
return partial(embedding_model.abatch_encode, batch_size=40)
|
||||
|
||||
def run_embedding():
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
return loop.run_until_complete(
|
||||
openai_embed(
|
||||
texts=texts,
|
||||
model=model,
|
||||
api_key=api_key,
|
||||
base_url=base_url.replace("/embeddings", ""),
|
||||
)
|
||||
)
|
||||
finally:
|
||||
loop.close()
|
||||
def _get_embedding_function(self, embed_info: dict):
|
||||
"""获取 embedding 函数"""
|
||||
config_dict = get_embedding_config(embed_info)
|
||||
embedding_model = OtherEmbedding(
|
||||
model=config_dict.get("model"),
|
||||
base_url=config_dict.get("base_url"),
|
||||
api_key=config_dict.get("api_key"),
|
||||
)
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
future = executor.submit(run_embedding)
|
||||
return future.result()
|
||||
|
||||
return embedding_function
|
||||
return partial(embedding_model.batch_encode, batch_size=40)
|
||||
|
||||
async def _get_milvus_collection(self, db_id: str):
|
||||
"""获取或创建 Milvus 集合"""
|
||||
@ -225,12 +210,21 @@ class MilvusKB(KnowledgeBase):
|
||||
logger.error(f"Traceback: {traceback.format_exc()}")
|
||||
return None
|
||||
|
||||
def _split_text_into_chunks(self, text: str, file_id: str, filename: str) -> List[Dict]:
|
||||
def _split_text_into_chunks(self, text: str, file_id: str, filename: str, params: dict) -> list[dict]:
|
||||
"""将文本分割成块"""
|
||||
return split_text_into_chunks(text, file_id, filename, self.chunk_size, self.chunk_overlap)
|
||||
# 检查是否使用QA分割模式
|
||||
use_qa_split = params.get('use_qa_split', False)
|
||||
|
||||
async def add_content(self, db_id: str, items: List[str],
|
||||
params: Optional[Dict] = None) -> List[Dict]:
|
||||
if use_qa_split:
|
||||
# 使用QA分割模式
|
||||
qa_separator = params.get('qa_separator', '\n\n\n')
|
||||
return split_text_into_qa_chunks(text, file_id, filename, qa_separator, params)
|
||||
else:
|
||||
# 使用传统分割模式
|
||||
return split_text_into_chunks(text, file_id, filename, params)
|
||||
|
||||
async def add_content(self, db_id: str, items: list[str],
|
||||
params: dict | None = {}) -> list[dict]:
|
||||
"""添加内容(文件/URL)"""
|
||||
if db_id not in self.databases_meta:
|
||||
raise ValueError(f"Database {db_id} not found")
|
||||
@ -240,7 +234,7 @@ class MilvusKB(KnowledgeBase):
|
||||
raise ValueError(f"Failed to get Milvus collection for {db_id}")
|
||||
|
||||
embed_info = self.databases_meta[db_id].get("embed_info", {})
|
||||
embedding_function = self._get_embedding_function(embed_info)
|
||||
embedding_function = self._get_async_embedding_function(embed_info)
|
||||
|
||||
content_type = params.get('content_type', 'file') if params else 'file'
|
||||
processed_items_info = []
|
||||
@ -250,7 +244,6 @@ class MilvusKB(KnowledgeBase):
|
||||
metadata = prepare_item_metadata(item, content_type, db_id)
|
||||
file_id = metadata["file_id"]
|
||||
filename = metadata["filename"]
|
||||
item_path = metadata["path"]
|
||||
|
||||
# 添加文件记录
|
||||
file_record = metadata.copy()
|
||||
@ -270,14 +263,14 @@ class MilvusKB(KnowledgeBase):
|
||||
markdown_content = await self._process_url_to_markdown(item, params=params)
|
||||
|
||||
# 分割文本成块
|
||||
chunks = self._split_text_into_chunks(markdown_content, file_id, filename)
|
||||
chunks = self._split_text_into_chunks(markdown_content, file_id, filename, params)
|
||||
logger.info(f"Split {filename} into {len(chunks)} chunks")
|
||||
|
||||
# 准备 Milvus 插入的数据
|
||||
if chunks:
|
||||
# 生成嵌入向量
|
||||
texts = [chunk["content"] for chunk in chunks]
|
||||
embeddings = embedding_function(texts)
|
||||
embeddings = await embedding_function(texts)
|
||||
|
||||
# 准备插入数据
|
||||
entities = [
|
||||
@ -390,7 +383,7 @@ class MilvusKB(KnowledgeBase):
|
||||
del self.files_meta[file_id]
|
||||
self._save_metadata()
|
||||
|
||||
async def get_file_info(self, db_id: str, file_id: str) -> Dict:
|
||||
async def get_file_info(self, db_id: str, file_id: str) -> dict:
|
||||
"""获取文件信息和chunks"""
|
||||
if file_id not in self.files_meta:
|
||||
raise Exception(f"File not found: {file_id}")
|
||||
@ -431,5 +424,5 @@ class MilvusKB(KnowledgeBase):
|
||||
try:
|
||||
if hasattr(self, 'connection_alias'):
|
||||
connections.disconnect(self.connection_alias)
|
||||
except:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@ -49,10 +49,10 @@ def select_embedding_model(model_id):
|
||||
raise ValueError("Local embedding model is not supported, please use other embedding models")
|
||||
|
||||
elif provider == "ollama":
|
||||
model = OllamaEmbedding(model_id)
|
||||
model = OllamaEmbedding(**config.embed_model_names[model_id])
|
||||
|
||||
else:
|
||||
model = OtherEmbedding(model_id)
|
||||
model = OtherEmbedding(**config.embed_model_names[model_id])
|
||||
|
||||
return model
|
||||
|
||||
|
||||
@ -12,14 +12,21 @@ from src.utils import hashstr, logger, get_docker_safe_url
|
||||
class BaseEmbeddingModel:
|
||||
embed_state = {}
|
||||
|
||||
def __init__(self, model_id):
|
||||
self.model_id = model_id
|
||||
self.info = config.embed_model_names[model_id]
|
||||
self.model = self.info["name"]
|
||||
self.dimension = self.info.get("dimension", None)
|
||||
self.url = get_docker_safe_url(self.info["base_url"])
|
||||
self.base_url = get_docker_safe_url(self.info["base_url"])
|
||||
self.api_key = os.getenv(self.info["api_key"], self.info["api_key"])
|
||||
def __init__(self, model=None, name=None, dimension=None, url=None, base_url=None, api_key=None):
|
||||
"""
|
||||
Args:
|
||||
model: 模型名称,冗余设计,同name
|
||||
name: 模型名称,冗余设计,同model
|
||||
dimension: 维度
|
||||
url: 请求URL,冗余设计,同base_url
|
||||
base_url: 基础URL,请求URL,冗余设计,同url
|
||||
api_key: 请求API密钥
|
||||
"""
|
||||
base_url = base_url or url
|
||||
self.model = model or name
|
||||
self.dimension = dimension
|
||||
self.base_url = get_docker_safe_url(base_url)
|
||||
self.api_key = os.getenv(api_key, api_key)
|
||||
|
||||
@abstractmethod
|
||||
def predict(self, message):
|
||||
@ -37,10 +44,10 @@ class BaseEmbeddingModel:
|
||||
async def aencode_queries(self, queries):
|
||||
return await asyncio.to_thread(self.encode_queries, queries)
|
||||
|
||||
async def abatch_encode(self, messages, batch_size=20):
|
||||
async def abatch_encode(self, messages, batch_size=40):
|
||||
return await asyncio.to_thread(self.batch_encode, messages, batch_size)
|
||||
|
||||
def batch_encode(self, messages, batch_size=20):
|
||||
def batch_encode(self, messages, batch_size=40):
|
||||
logger.info(f"Batch encoding {len(messages)} messages")
|
||||
data = []
|
||||
|
||||
@ -70,9 +77,9 @@ class OllamaEmbedding(BaseEmbeddingModel):
|
||||
Ollama Embedding Model
|
||||
"""
|
||||
|
||||
def __init__(self, model_id) -> None:
|
||||
super().__init__(model_id)
|
||||
self.url = self.url or get_docker_safe_url("http://localhost:11434/api/embed")
|
||||
def __init__(self, **kwargs) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self.base_url = self.base_url or get_docker_safe_url("http://localhost:11434/api/embed")
|
||||
|
||||
def predict(self, message: list[str] | str):
|
||||
if isinstance(message, str):
|
||||
@ -82,7 +89,7 @@ class OllamaEmbedding(BaseEmbeddingModel):
|
||||
"model": self.model,
|
||||
"input": message,
|
||||
}
|
||||
response = requests.request("POST", self.url, json=payload)
|
||||
response = requests.request("POST", self.base_url, json=payload)
|
||||
response = json.loads(response.text)
|
||||
assert response.get("embeddings"), f"Ollama Embedding failed: {response}"
|
||||
return response["embeddings"]
|
||||
@ -90,8 +97,8 @@ class OllamaEmbedding(BaseEmbeddingModel):
|
||||
|
||||
class OtherEmbedding(BaseEmbeddingModel):
|
||||
|
||||
def __init__(self, model_id) -> None:
|
||||
super().__init__(model_id)
|
||||
def __init__(self, **kwargs) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self.headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json"
|
||||
@ -99,7 +106,7 @@ class OtherEmbedding(BaseEmbeddingModel):
|
||||
|
||||
def predict(self, message):
|
||||
payload = self.build_payload(message)
|
||||
response = requests.request("POST", self.url, json=payload, headers=self.headers)
|
||||
response = requests.request("POST", self.base_url, json=payload, headers=self.headers)
|
||||
response = json.loads(response.text)
|
||||
assert response["data"], f"Other Embedding failed: {response}"
|
||||
data = [a["embedding"] for a in response["data"]]
|
||||
|
||||
@ -261,7 +261,7 @@ class OCRPlugin:
|
||||
error_detail = "Unknown error"
|
||||
try:
|
||||
error_detail = health_check_response.json()
|
||||
except:
|
||||
except Exception:
|
||||
error_detail = health_check_response.text
|
||||
|
||||
raise OCRServiceException(
|
||||
@ -322,7 +322,7 @@ class OCRPlugin:
|
||||
error_detail = "Unknown error"
|
||||
try:
|
||||
error_detail = health_check_response.json()
|
||||
except:
|
||||
except Exception:
|
||||
error_detail = health_check_response.text
|
||||
|
||||
raise OCRServiceException(
|
||||
|
||||
@ -327,7 +327,7 @@ def analyze_folder(input_dir: str, output_dir: str, base_url: str = "http://loca
|
||||
files.append(file_path)
|
||||
|
||||
if not files:
|
||||
print(f"⚠️ 没有找到支持的文件")
|
||||
print("⚠️ 没有找到支持的文件")
|
||||
return
|
||||
|
||||
print(f"📁 找到 {len(files)} 个文件")
|
||||
@ -381,4 +381,4 @@ if __name__ == "__main__":
|
||||
"""批量分析文件夹"""
|
||||
analyze_folder(input_dir, output_dir, base_url)
|
||||
|
||||
app()
|
||||
app()
|
||||
|
||||
@ -120,6 +120,12 @@ EMBED_MODEL_INFO:
|
||||
base_url: https://api.siliconflow.cn/v1/embeddings
|
||||
api_key: SILICONFLOW_API_KEY
|
||||
|
||||
siliconflow/Qwen/Qwen3-Embedding-0.6B:
|
||||
name: Qwen/Qwen3-Embedding-0.6B
|
||||
dimension: 1024
|
||||
base_url: https://api.siliconflow.cn/v1/embeddings
|
||||
api_key: SILICONFLOW_API_KEY
|
||||
|
||||
vllm/Qwen/Qwen3-Embedding-0.6B:
|
||||
name: Qwen3-Embedding-0.6B
|
||||
dimension: 1024
|
||||
|
||||
@ -47,6 +47,9 @@ def hashstr(input_string, length=None, with_salt=False):
|
||||
|
||||
|
||||
def get_docker_safe_url(base_url):
|
||||
if not base_url:
|
||||
return base_url
|
||||
|
||||
if os.getenv("RUNNING_IN_DOCKER") == "true":
|
||||
# 替换所有可能的本地地址形式
|
||||
base_url = base_url.replace("http://localhost", "http://host.docker.internal")
|
||||
|
||||
@ -36,13 +36,7 @@ export const databaseApi = {
|
||||
*/
|
||||
createDatabase: async (databaseData) => {
|
||||
checkAdminPermission()
|
||||
return apiPost('/api/knowledge/databases', {
|
||||
database_name: databaseData.database_name,
|
||||
description: databaseData.description,
|
||||
embed_model_name: databaseData.embed_model_name,
|
||||
kb_type: databaseData.kb_type || 'lightrag',
|
||||
...databaseData.extra_config
|
||||
}, {}, true)
|
||||
return apiPost('/api/knowledge/databases', databaseData, {}, true)
|
||||
},
|
||||
|
||||
/**
|
||||
|
||||
@ -2,39 +2,39 @@
|
||||
/* color palette from <https://github.com/vuejs/theme> */
|
||||
/* https://material-foundation.github.io/material-theme-builder/ */
|
||||
:root {
|
||||
--main-1000: #002A36;
|
||||
--main-900: #003A51;
|
||||
--main-800: #004F69;
|
||||
--main-700: #00637F;
|
||||
--main-600: #1B7796;
|
||||
--main-500: #2C86A8;
|
||||
--main-400: #4e99b9;
|
||||
--main-300: #6AADCB;
|
||||
--main-200: #8CC6E1;
|
||||
--main-100: #ABE0F7;
|
||||
--main-50: #CDF5FF;
|
||||
--main-25: #E6FAFF;
|
||||
--main-10: #F5FDFF;
|
||||
--main-1000: #011A26;
|
||||
--main-900: #034A51;
|
||||
--main-800: #034F69;
|
||||
--main-700: #03637F;
|
||||
--main-600: #1D7796;
|
||||
--main-500: #2F86A8;
|
||||
--main-400: #5099B9;
|
||||
--main-300: #6CADCB;
|
||||
--main-200: #8EC6E1;
|
||||
--main-100: #ADE0F7;
|
||||
--main-50: #CFF5FF;
|
||||
--main-25: #E8FAFF;
|
||||
--main-10: #F8FDFF;
|
||||
--main-5: #FAFCFD;
|
||||
|
||||
--gray-10000: #000000;
|
||||
--gray-2000: #0C1214;
|
||||
--gray-1000: #171C1F;
|
||||
--gray-900: #212729;
|
||||
--gray-800: #42484A;
|
||||
--gray-2000: #0F0F0F;
|
||||
--gray-1000: #171717;
|
||||
--gray-900: #212121;
|
||||
--gray-800: #424242;
|
||||
--gray-700: #616161;
|
||||
--gray-600: #8C9193;
|
||||
--gray-500: #A7ACAD;
|
||||
--gray-400: #C4C7C8;
|
||||
--gray-300: #DFE3E4;
|
||||
--gray-200: #EFF1F2;
|
||||
--gray-100: #F8FAFB;
|
||||
--gray-50: #FBFDFE;
|
||||
--gray-25: #FCFEFF;
|
||||
--gray-10: #FDFEFF;
|
||||
--gray-600: #8C8C8C;
|
||||
--gray-500: #A7A7A7;
|
||||
--gray-400: #C4C4C4;
|
||||
--gray-300: #EFEFEF;
|
||||
--gray-200: #EFEFEF;
|
||||
--gray-100: #F8F8F8;
|
||||
--gray-50: #FBFBFB;
|
||||
--gray-25: #FCFCFC;
|
||||
--gray-10: #FDFDFD;
|
||||
--gray-0: #FFFFFF;
|
||||
|
||||
--main-color: #1c6586;
|
||||
--main-color: #016179;
|
||||
--main-color-dark: #004d5c;
|
||||
--main-color-text: #104461;
|
||||
--main-light-1: #0076AB;
|
||||
@ -47,6 +47,7 @@
|
||||
--secondry-color: #4e616d;
|
||||
--error-color: #ba1a1a;
|
||||
|
||||
|
||||
--bg-sider: var(--main-5);
|
||||
--color-text: var(--c-black);
|
||||
|
||||
|
||||
@ -261,7 +261,7 @@ const toggleToolCall = (toolCallId) => {
|
||||
}
|
||||
|
||||
.ant-collapse-expand-icon {
|
||||
color: var(--main-600);
|
||||
color: var(--main-color);
|
||||
}
|
||||
}
|
||||
|
||||
@ -351,7 +351,7 @@ const toggleToolCall = (toolCallId) => {
|
||||
}
|
||||
|
||||
.anticon {
|
||||
color: var(--main-600);
|
||||
color: var(--main-color);
|
||||
font-size: 16px;
|
||||
}
|
||||
|
||||
|
||||
@ -455,8 +455,8 @@ const printAgentConfig = async () => {
|
||||
border-color: var(--main-color);
|
||||
|
||||
&:hover {
|
||||
background-color: var(--main-600);
|
||||
border-color: var(--main-600);
|
||||
background-color: var(--main-color);
|
||||
border-color: var(--main-color);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -264,7 +264,7 @@ const isEmptyAndLoading = computed(() => {
|
||||
position: relative;
|
||||
|
||||
.anticon {
|
||||
color: var(--main-600);
|
||||
color: var(--main-color);
|
||||
}
|
||||
|
||||
.step-badge {
|
||||
|
||||
@ -211,11 +211,11 @@ onMounted(() => {
|
||||
|
||||
&:hover {
|
||||
background-color: var(--main-10);
|
||||
color: var(--main-600);
|
||||
color: var(--main-color);
|
||||
}
|
||||
|
||||
&.active {
|
||||
color: var(--main-600);
|
||||
color: var(--main-color);
|
||||
border: 1px solid var(--main-500);
|
||||
background-color: var(--main-10);
|
||||
}
|
||||
@ -241,7 +241,7 @@ button.ant-btn-icon-only {
|
||||
font-size: 14px;
|
||||
|
||||
&:hover {
|
||||
background-color: var(--main-600);
|
||||
background-color: var(--main-color);
|
||||
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.15);
|
||||
color: white;
|
||||
}
|
||||
|
||||
@ -245,7 +245,7 @@ const totalRelations = computed(() => graphData.value.edges.length)
|
||||
}
|
||||
|
||||
.relation-type {
|
||||
color: var(--main-600);
|
||||
color: var(--main-color);
|
||||
font-weight: 500;
|
||||
background: var(--gray-100);
|
||||
padding: 2px 6px;
|
||||
|
||||
@ -134,7 +134,7 @@ const formatDate = (dateString) => {
|
||||
font-weight: 500;
|
||||
|
||||
&:hover {
|
||||
color: var(--main-600);
|
||||
color: var(--main-color);
|
||||
text-decoration: underline;
|
||||
}
|
||||
}
|
||||
|
||||
@ -251,7 +251,7 @@ div.header, #app-router-view {
|
||||
flex: 0 0 var(--header-width);
|
||||
justify-content: flex-start;
|
||||
align-items: center;
|
||||
background-color: var(--gray-100);
|
||||
background-color: var(--main-10);
|
||||
height: 100%;
|
||||
width: var(--header-width);
|
||||
border-right: 1px solid var(--gray-300);
|
||||
@ -353,7 +353,7 @@ div.header, #app-router-view {
|
||||
}
|
||||
&.active {
|
||||
font-weight: bold;
|
||||
color: var(--main-600);
|
||||
color: var(--main-color);
|
||||
background-color: white;
|
||||
border: 1px solid white;
|
||||
}
|
||||
@ -471,7 +471,7 @@ div.header, #app-router-view {
|
||||
font-size: 16px;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.5px;
|
||||
color: var(--main-600);
|
||||
color: var(--main-color);
|
||||
white-space: nowrap;
|
||||
}
|
||||
}
|
||||
@ -514,7 +514,7 @@ div.header, #app-router-view {
|
||||
}
|
||||
|
||||
&.active {
|
||||
color: var(--main-600);
|
||||
color: var(--main-color);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
<a-select
|
||||
v-model:value="selectedAgentId"
|
||||
class="agent-list"
|
||||
style="width: 300px"
|
||||
style="width: 240px"
|
||||
@change="selectAgent"
|
||||
>
|
||||
<a-select-option
|
||||
@ -105,20 +105,6 @@
|
||||
:placeholder="getPlaceholder(key, value)"
|
||||
/>
|
||||
|
||||
<!-- 数据类型匹配 -->
|
||||
<a-switch
|
||||
v-else-if="typeof agentConfig[key] === 'boolean'"
|
||||
v-model:checked="agentConfig[key]"
|
||||
/>
|
||||
<!-- 单选 -->
|
||||
<a-select
|
||||
v-else-if="value?.options && (value?.type === 'str' || value?.type === 'select')"
|
||||
v-model:value="agentConfig[key]"
|
||||
>
|
||||
<a-select-option v-for="option in value.options" :key="option" :value="option">
|
||||
{{ option.label || option }}
|
||||
</a-select-option>
|
||||
</a-select>
|
||||
<!-- 工具选择特殊处理 -->
|
||||
<div v-else-if="key === 'tools'" class="tools-selector">
|
||||
<div class="tools-summary">
|
||||
@ -138,6 +124,25 @@
|
||||
</a-tag>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 数据类型匹配 -->
|
||||
|
||||
<!-- 布尔类型 -->
|
||||
<a-switch
|
||||
v-else-if="typeof agentConfig[key] === 'boolean'"
|
||||
v-model:checked="agentConfig[key]"
|
||||
/>
|
||||
|
||||
<!-- 单选 -->
|
||||
<a-select
|
||||
v-else-if="value?.options && (value?.type === 'str' || value?.type === 'select')"
|
||||
v-model:value="agentConfig[key]"
|
||||
>
|
||||
<a-select-option v-for="option in value.options" :key="option" :value="option">
|
||||
{{ option.label || option }}
|
||||
</a-select-option>
|
||||
</a-select>
|
||||
|
||||
<!-- 多选 -->
|
||||
<div v-else-if="value?.options && value?.type === 'list'" class="multi-select-cards">
|
||||
<div class="multi-select-label">
|
||||
@ -167,12 +172,14 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 数字 -->
|
||||
<a-input-number
|
||||
v-else-if="value?.type === 'number'"
|
||||
v-model:value="agentConfig[key]"
|
||||
:placeholder="getPlaceholder(key, value)"
|
||||
/>
|
||||
|
||||
<!-- 滑块 -->
|
||||
<a-slider
|
||||
v-else-if="value?.type === 'slider'"
|
||||
@ -182,6 +189,7 @@
|
||||
:step="value.step"
|
||||
style="max-width: 300px;"
|
||||
/>
|
||||
|
||||
<!-- 其他类型 -->
|
||||
<a-input
|
||||
v-else
|
||||
@ -408,10 +416,10 @@ const clearSelection = (key) => {
|
||||
// 工具选择相关方法
|
||||
const openToolsModal = async () => {
|
||||
try {
|
||||
// 加载可用工具列表
|
||||
const response = await toolsApi.getTools();
|
||||
// 将工具对象转换为数组格式,保留id和name信息
|
||||
availableTools.value = Object.values(response.tools || {});
|
||||
// 如果工具列表还没有加载,则加载
|
||||
if (availableTools.value.length === 0) {
|
||||
await loadAvailableTools();
|
||||
}
|
||||
|
||||
// 初始化已选择的工具
|
||||
selectedTools.value = [...(agentConfig.value.tools || [])];
|
||||
@ -419,8 +427,8 @@ const openToolsModal = async () => {
|
||||
// 打开弹窗
|
||||
state.toolsModalOpen = true;
|
||||
} catch (error) {
|
||||
console.error('加载工具列表失败:', error);
|
||||
message.error('加载工具列表失败');
|
||||
console.error('打开工具选择弹窗失败:', error);
|
||||
message.error('打开工具选择弹窗失败');
|
||||
}
|
||||
};
|
||||
|
||||
@ -605,12 +613,24 @@ const selectAgent = (agentId) => {
|
||||
loadAgentConfig();
|
||||
};
|
||||
|
||||
// 加载工具列表
|
||||
const loadAvailableTools = async () => {
|
||||
try {
|
||||
const response = await toolsApi.getTools();
|
||||
availableTools.value = Object.values(response.tools || {});
|
||||
} catch (error) {
|
||||
console.error('加载工具列表失败:', error);
|
||||
}
|
||||
};
|
||||
|
||||
// 初始化
|
||||
onMounted(async () => {
|
||||
// 获取默认智能体
|
||||
await fetchDefaultAgent();
|
||||
// 获取智能体列表
|
||||
await fetchAgents();
|
||||
// 加载工具列表
|
||||
await loadAvailableTools();
|
||||
|
||||
// 恢复上次选择的智能体
|
||||
const lastSelectedAgent = localStorage.getItem('last-selected-agent');
|
||||
@ -677,7 +697,7 @@ const toggleConf = () => {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 0 16px;
|
||||
padding: 2px 8px;
|
||||
|
||||
.header-left,
|
||||
.header-right,
|
||||
@ -784,7 +804,7 @@ const toggleConf = () => {
|
||||
border-color: var(--main-color);
|
||||
|
||||
&:disabled {
|
||||
color: var(--main-600);
|
||||
color: var(--main-color);
|
||||
background-color: var(--main-light-4);
|
||||
cursor: not-allowed;
|
||||
opacity: 0.7;
|
||||
@ -860,7 +880,7 @@ const toggleConf = () => {
|
||||
transition: all 0.2s ease;
|
||||
|
||||
&:hover {
|
||||
background: var(--main-600);
|
||||
background: var(--main-color);
|
||||
transform: translateY(-1px);
|
||||
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.15);
|
||||
}
|
||||
@ -1020,7 +1040,7 @@ const toggleConf = () => {
|
||||
border: none;
|
||||
color: #fff;
|
||||
&:hover {
|
||||
background: var(--main-600);
|
||||
background: var(--main-color);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -3,9 +3,10 @@
|
||||
<HeaderComponent
|
||||
:title="database.name || '数据库信息加载中'"
|
||||
:loading="state.databaseLoading"
|
||||
class="database-info-header"
|
||||
>
|
||||
<template #left>
|
||||
<a-button type="link" @click="backToDatabase" :style="{ padding: '0px', color: 'inherit' }">
|
||||
<a-button @click="backToDatabase">
|
||||
<LeftOutlined />
|
||||
</a-button>
|
||||
</template>
|
||||
@ -74,6 +75,14 @@
|
||||
<a-input-number v-model:value="tempChunkParams.chunk_overlap" :min="0" :max="1000" style="width: 100%;" />
|
||||
<p class="param-description">相邻文本片段间的重叠字符数</p>
|
||||
</a-form-item>
|
||||
<a-form-item v-if="isQaSplitSupported" label="QA分割模式" name="use_qa_split">
|
||||
<a-switch v-model:checked="tempChunkParams.use_qa_split" />
|
||||
<p class="param-description">启用后将按QA对分割,忽略上述chunk大小设置</p>
|
||||
</a-form-item>
|
||||
<a-form-item v-if="tempChunkParams.use_qa_split && isQaSplitSupported" label="QA分隔符" name="qa_separator">
|
||||
<a-input v-model:value="tempChunkParams.qa_separator" placeholder="输入QA分隔符" style="width: 100%;" />
|
||||
<p class="param-description">用于分割不同QA对的分隔符</p>
|
||||
</a-form-item>
|
||||
</a-form>
|
||||
</div>
|
||||
</a-modal>
|
||||
@ -139,7 +148,37 @@
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</a-form-item>
|
||||
</a-form-item>
|
||||
</a-form>
|
||||
</div>
|
||||
|
||||
<div class="qa-split-config" v-if="isQaSplitSupported">
|
||||
<a-form layout="horizontal">
|
||||
<a-form-item label="QA分割模式" name="use_qa_split">
|
||||
<div class="toggle-controls">
|
||||
<a-switch
|
||||
v-model:checked="chunkParams.use_qa_split"
|
||||
style="margin-right: 12px;"
|
||||
/>
|
||||
<span class="param-description">
|
||||
{{ chunkParams.use_qa_split ? '启用QA分割(忽略chunk大小设置)' : '使用普通分割模式' }}
|
||||
</span>
|
||||
</div>
|
||||
</a-form-item>
|
||||
<a-form-item
|
||||
v-if="chunkParams.use_qa_split"
|
||||
label="QA分隔符"
|
||||
name="qa_separator"
|
||||
>
|
||||
<a-input
|
||||
v-model:value="chunkParams.qa_separator"
|
||||
placeholder="输入QA分隔符"
|
||||
style="width: 200px; margin-right: 12px;"
|
||||
/>
|
||||
<span class="param-description">
|
||||
用于分割不同QA对的分隔符,默认为3个换行符
|
||||
</span>
|
||||
</a-form-item>
|
||||
</a-form>
|
||||
</div>
|
||||
|
||||
@ -235,7 +274,7 @@
|
||||
<div class="info-item">
|
||||
<label>处理状态:</label>
|
||||
<span class="status-badge" :class="selectedFile.status">
|
||||
{{ getStatusText(selectedFile.status) }}
|
||||
{{ getStatusText(selectedFile.status) }} - {{ selectedFile.lines.length }} 行
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
@ -267,23 +306,16 @@
|
||||
<div class="unified-layout">
|
||||
<div class="left-panel" :style="{ width: leftPanelWidth + '%' }">
|
||||
<div class="panel-header">
|
||||
<h3 style="margin-left: 8px;">共 {{ database.files ? Object.keys(database.files).length : 0 }} 个文件</h3>
|
||||
<div class="panel-actions">
|
||||
|
||||
<div class="search-container">
|
||||
<a-button
|
||||
type="secondary"
|
||||
@click="showAddFilesModal"
|
||||
:loading="state.refrashing"
|
||||
:icon="h(PlusOutlined)"
|
||||
title="添加文件"
|
||||
/>
|
||||
>添加文件</a-button>
|
||||
</div>
|
||||
<div class="panel-actions">
|
||||
<div class="refresh-group">
|
||||
<a-switch
|
||||
v-model:checked="state.autoRefresh"
|
||||
@change="toggleAutoRefresh"
|
||||
size="small"
|
||||
title="自动刷新文件状态"
|
||||
/>
|
||||
<a-button
|
||||
type="text"
|
||||
@click="handleRefresh"
|
||||
@ -291,7 +323,21 @@
|
||||
:icon="h(ReloadOutlined)"
|
||||
title="刷新"
|
||||
/>
|
||||
<a-switch
|
||||
v-model:checked="state.autoRefresh"
|
||||
@change="toggleAutoRefresh"
|
||||
size="small"
|
||||
title="自动刷新文件状态"
|
||||
/>
|
||||
</div>
|
||||
<a-input
|
||||
v-model:value="filenameFilter"
|
||||
placeholder="搜索文件名"
|
||||
size="small"
|
||||
style="width: 120px; margin-right: 8px; border-radius: 6px; padding: 4px 8px;"
|
||||
allow-clear
|
||||
@change="onFilterChange"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -310,7 +356,7 @@
|
||||
|
||||
<a-table
|
||||
:columns="columnsCompact"
|
||||
:data-source="Object.values(database.files || {})"
|
||||
:data-source="filteredFiles"
|
||||
row-key="file_id"
|
||||
class="my-table-compact"
|
||||
size="small"
|
||||
@ -320,6 +366,9 @@
|
||||
selectedRowKeys: selectedRowKeys,
|
||||
onChange: onSelectChange,
|
||||
getCheckboxProps: getCheckboxProps
|
||||
}"
|
||||
:locale="{
|
||||
emptyText: emptyText
|
||||
}">
|
||||
<template #bodyCell="{ column, text, record }">
|
||||
<a-button v-if="column.key === 'filename'" class="main-btn" type="link" @click="openFileDetail(record)">{{ text }}</a-button>
|
||||
@ -567,6 +616,9 @@ const database = ref({});
|
||||
const fileList = ref([]);
|
||||
const selectedFile = ref(null);
|
||||
|
||||
// 文件名过滤
|
||||
const filenameFilter = ref('');
|
||||
|
||||
// 查询测试
|
||||
const queryText = ref('');
|
||||
const queryResult = ref(null)
|
||||
@ -597,6 +649,12 @@ const isGraphSupported = computed(() => {
|
||||
return kbType === 'lightrag';
|
||||
});
|
||||
|
||||
// 计算属性:是否支持QA分割
|
||||
const isQaSplitSupported = computed(() => {
|
||||
const kbType = database.value.kb_type?.toLowerCase();
|
||||
return kbType === 'chroma' || kbType === 'milvus';
|
||||
});
|
||||
|
||||
// 切换图谱最大化状态
|
||||
const toggleGraphMaximize = () => {
|
||||
state.isGraphMaximized = !state.isGraphMaximized;
|
||||
@ -962,6 +1020,13 @@ const getDatabaseInfo = () => {
|
||||
databaseApi.getDatabaseInfo(db_id)
|
||||
.then(async data => {
|
||||
database.value = data
|
||||
|
||||
// 如果当前知识库类型不支持QA分割,重置相关参数
|
||||
const kbType = data.kb_type?.toLowerCase();
|
||||
if (kbType !== 'chroma' && kbType !== 'milvus') {
|
||||
chunkParams.value.use_qa_split = false;
|
||||
}
|
||||
|
||||
// 加载查询参数
|
||||
await loadQueryParams()
|
||||
resolve(data)
|
||||
@ -1069,6 +1134,8 @@ const chunkParams = ref({
|
||||
chunk_size: 1000,
|
||||
chunk_overlap: 200,
|
||||
enable_ocr: 'disable',
|
||||
use_qa_split: false,
|
||||
qa_separator: '\n\n\n',
|
||||
})
|
||||
|
||||
// "生成分块" - 新的统一方法
|
||||
@ -1124,6 +1191,14 @@ watch(() => route.params.database_id, async (newId) => {
|
||||
}
|
||||
);
|
||||
|
||||
// 监听数据库信息变化,重置过滤条件
|
||||
watch(() => database.value, () => {
|
||||
// 当数据库信息更新时,重置过滤条件和分页
|
||||
filenameFilter.value = '';
|
||||
paginationCompact.value.current = 1;
|
||||
selectedRowKeys.value = [];
|
||||
}, { deep: true });
|
||||
|
||||
|
||||
// 添加更多示例查询
|
||||
const queryExamples = ref([
|
||||
@ -1216,8 +1291,21 @@ onMounted(() => {
|
||||
if (resizeHandleHorizontal.value) {
|
||||
resizeHandleHorizontal.value.addEventListener('mousedown', handleMouseDownHorizontal);
|
||||
}
|
||||
|
||||
// 添加键盘事件监听
|
||||
document.addEventListener('keydown', handleKeyDown);
|
||||
});
|
||||
|
||||
// 键盘事件处理
|
||||
const handleKeyDown = (e) => {
|
||||
// 当按下ESC键时清空过滤条件
|
||||
if (e.key === 'Escape' && filenameFilter.value) {
|
||||
filenameFilter.value = '';
|
||||
selectedRowKeys.value = [];
|
||||
paginationCompact.value.current = 1;
|
||||
}
|
||||
};
|
||||
|
||||
// 清理事件监听
|
||||
onUnmounted(() => {
|
||||
stopAutoRefresh();
|
||||
@ -1231,6 +1319,7 @@ onUnmounted(() => {
|
||||
document.removeEventListener('mouseup', handleMouseUp);
|
||||
document.removeEventListener('mousemove', handleMouseMoveHorizontal);
|
||||
document.removeEventListener('mouseup', handleMouseUpHorizontal);
|
||||
document.removeEventListener('keydown', handleKeyDown);
|
||||
});
|
||||
|
||||
const uploadMode = ref('file');
|
||||
@ -1258,6 +1347,10 @@ const chunkData = () => {
|
||||
|
||||
addFiles(urls, 'url');
|
||||
}
|
||||
setTimeout(() => {
|
||||
addFilesModalVisible.value = false;
|
||||
getDatabaseInfo();
|
||||
}, 1000);
|
||||
}
|
||||
|
||||
const getAuthHeaders = () => {
|
||||
@ -1282,6 +1375,8 @@ const chunkConfigModalVisible = ref(false);
|
||||
const tempChunkParams = ref({
|
||||
chunk_size: 1000,
|
||||
chunk_overlap: 200,
|
||||
use_qa_split: false,
|
||||
qa_separator: '\n\n\n',
|
||||
});
|
||||
|
||||
// 添加文件弹窗
|
||||
@ -1328,6 +1423,8 @@ const showChunkConfigModal = () => {
|
||||
tempChunkParams.value = {
|
||||
chunk_size: chunkParams.value.chunk_size,
|
||||
chunk_overlap: chunkParams.value.chunk_overlap,
|
||||
use_qa_split: isQaSplitSupported.value ? chunkParams.value.use_qa_split : false,
|
||||
qa_separator: chunkParams.value.qa_separator,
|
||||
};
|
||||
chunkConfigModalVisible.value = true;
|
||||
};
|
||||
@ -1336,6 +1433,13 @@ const showChunkConfigModal = () => {
|
||||
const handleChunkConfigSubmit = () => {
|
||||
chunkParams.value.chunk_size = tempChunkParams.value.chunk_size;
|
||||
chunkParams.value.chunk_overlap = tempChunkParams.value.chunk_overlap;
|
||||
// 只有支持QA分割的知识库类型才保存QA分割配置
|
||||
if (isQaSplitSupported.value) {
|
||||
chunkParams.value.use_qa_split = tempChunkParams.value.use_qa_split;
|
||||
chunkParams.value.qa_separator = tempChunkParams.value.qa_separator;
|
||||
} else {
|
||||
chunkParams.value.use_qa_split = false;
|
||||
}
|
||||
chunkConfigModalVisible.value = false;
|
||||
message.success('分块参数配置已更新');
|
||||
};
|
||||
@ -1410,11 +1514,29 @@ const columnsCompact = [
|
||||
{ title: '', key: 'action', dataIndex: 'file_id', width: 40, align: 'center' }
|
||||
];
|
||||
|
||||
// 过滤后的文件列表
|
||||
const filteredFiles = computed(() => {
|
||||
const files = Object.values(database.value.files || {});
|
||||
if (!filenameFilter.value.trim()) {
|
||||
return files;
|
||||
}
|
||||
|
||||
const filterText = filenameFilter.value.toLowerCase().trim();
|
||||
return files.filter(file =>
|
||||
file.filename && file.filename.toLowerCase().includes(filterText)
|
||||
);
|
||||
});
|
||||
|
||||
// 空状态文本
|
||||
const emptyText = computed(() => {
|
||||
return filenameFilter.value ? `没有找到包含"${filenameFilter.value}"的文件` : '暂无文件';
|
||||
});
|
||||
|
||||
// 紧凑分页配置
|
||||
const paginationCompact = ref({
|
||||
pageSize: 20,
|
||||
current: 1,
|
||||
total: computed(() => database.value?.files?.length || 0),
|
||||
total: computed(() => filteredFiles.value.length),
|
||||
showSizeChanger: false,
|
||||
onChange: (page) => paginationCompact.value.current = page,
|
||||
showTotal: (total) => `${total}`,
|
||||
@ -1432,14 +1554,28 @@ const getCheckboxProps = (record) => ({
|
||||
disabled: state.lock || record.status === 'processing' || record.status === 'waiting',
|
||||
});
|
||||
|
||||
// 过滤相关方法
|
||||
const onFilterSearch = (value) => {
|
||||
filenameFilter.value = value;
|
||||
// 重置分页到第一页
|
||||
paginationCompact.value.current = 1;
|
||||
// 清空选中的行,因为过滤后选中的行可能不在当前视图中
|
||||
selectedRowKeys.value = [];
|
||||
};
|
||||
|
||||
const onFilterChange = (e) => {
|
||||
filenameFilter.value = e.target.value;
|
||||
// 重置分页到第一页
|
||||
paginationCompact.value.current = 1;
|
||||
// 清空选中的行,因为过滤后选中的行可能不在当前视图中
|
||||
selectedRowKeys.value = [];
|
||||
};
|
||||
|
||||
// 计算是否可以批量删除
|
||||
const canBatchDelete = computed(() => {
|
||||
const files = database.value.files || {};
|
||||
return selectedRowKeys.value.some(key => {
|
||||
const file = files[key];
|
||||
return !(state.lock || file.status === 'processing' || file.status === 'waiting');
|
||||
const file = filteredFiles.value.find(f => f.file_id === key);
|
||||
return file && !(state.lock || file.status === 'processing' || file.status === 'waiting');
|
||||
});
|
||||
});
|
||||
|
||||
@ -1528,6 +1664,11 @@ const getKbTypeColor = (type) => {
|
||||
</script>
|
||||
|
||||
<style lang="less" scoped>
|
||||
.database-info-header {
|
||||
padding: 8px 16px 6px 16px;
|
||||
height: 50px;
|
||||
}
|
||||
|
||||
.database-info {
|
||||
margin: 8px 0 0;
|
||||
display: flex;
|
||||
@ -1549,11 +1690,6 @@ const getKbTypeColor = (type) => {
|
||||
}
|
||||
}
|
||||
|
||||
.header-container {
|
||||
padding: 8px 16px;
|
||||
height: 54px;
|
||||
}
|
||||
|
||||
.header-info {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
@ -1696,7 +1832,6 @@ const getKbTypeColor = (type) => {
|
||||
}
|
||||
|
||||
.content-lines {
|
||||
max-height: 400px;
|
||||
overflow-y: auto;
|
||||
border: 1px solid var(--gray-200);
|
||||
border-radius: 4px;
|
||||
@ -1846,6 +1981,54 @@ const getKbTypeColor = (type) => {
|
||||
}
|
||||
}
|
||||
|
||||
.qa-split-config {
|
||||
margin-bottom: 16px;
|
||||
padding: 12px 16px;
|
||||
background-color: var(--main-light-6);
|
||||
border-radius: 8px;
|
||||
border: 1px solid var(--main-light-3);
|
||||
|
||||
.ant-form-item {
|
||||
margin-bottom: 12px;
|
||||
|
||||
&:last-child {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.ant-form-item-label {
|
||||
color: var(--gray-800);
|
||||
font-weight: 500;
|
||||
}
|
||||
}
|
||||
|
||||
.toggle-controls {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.param-description {
|
||||
color: var(--gray-600);
|
||||
font-size: 12px;
|
||||
margin-left: 0;
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
.ant-input {
|
||||
border-color: var(--main-light-3);
|
||||
|
||||
&:hover,
|
||||
&:focus {
|
||||
border-color: var(--main-color);
|
||||
}
|
||||
}
|
||||
|
||||
.ant-switch {
|
||||
&.ant-switch-checked {
|
||||
background-color: var(--main-color);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.upload {
|
||||
margin-bottom: 20px;
|
||||
|
||||
@ -2027,14 +2210,52 @@ const getKbTypeColor = (type) => {
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
|
||||
.search-container {
|
||||
position: relative;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
|
||||
.search-hint {
|
||||
position: absolute;
|
||||
right: -60px;
|
||||
top: 50%;
|
||||
transform: translateY(-50%);
|
||||
font-size: 10px;
|
||||
color: var(--gray-500);
|
||||
white-space: nowrap;
|
||||
}
|
||||
}
|
||||
|
||||
.ant-input-search {
|
||||
.ant-input {
|
||||
font-size: 12px;
|
||||
border-radius: 4px;
|
||||
|
||||
&:hover,
|
||||
&:focus {
|
||||
border-color: var(--main-color);
|
||||
}
|
||||
}
|
||||
|
||||
.ant-btn {
|
||||
font-size: 12px;
|
||||
border-radius: 0 4px 4px 0;
|
||||
|
||||
&:hover {
|
||||
border-color: var(--main-color);
|
||||
color: var(--main-color);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.refresh-group {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
background: var(--gray-50);
|
||||
padding: 2px 6px;
|
||||
border-radius: 12px;
|
||||
border: 1px solid var(--gray-200);
|
||||
background: var(--gray-100);
|
||||
padding: 0px 6px;
|
||||
border-radius: 8px;
|
||||
border: 1px solid var(--gray-300);
|
||||
}
|
||||
|
||||
.ant-btn {
|
||||
@ -2119,7 +2340,7 @@ const getKbTypeColor = (type) => {
|
||||
padding: 8px;
|
||||
height: 36px;
|
||||
flex-shrink: 0;
|
||||
background-color: var(--gray-200);
|
||||
background-color: var(--gray-100);
|
||||
backdrop-filter: blur(4px);
|
||||
position: sticky;
|
||||
top: 0;
|
||||
|
||||
@ -12,61 +12,53 @@
|
||||
|
||||
<!-- 知识库类型选择 -->
|
||||
<h3>知识库类型<span style="color: var(--error-color)">*</span></h3>
|
||||
<a-select v-model:value="newDatabase.kb_type" @change="handleKbTypeChange" style="width: 100%;" size="large">
|
||||
<a-select-option v-for="(typeInfo, typeKey) in supportedKbTypes" :key="typeKey" :value="typeKey">
|
||||
<div class="kb-type-option">
|
||||
<div class="type-header">
|
||||
<component :is="getKbTypeIcon(typeKey)" class="type-icon" />
|
||||
<span class="type-title">{{ getKbTypeLabel(typeKey) }}</span>
|
||||
</div>
|
||||
<div class="type-desc">{{ typeInfo.description }}</div>
|
||||
<div class="kb-type-cards">
|
||||
<div
|
||||
v-for="(typeInfo, typeKey) in supportedKbTypes"
|
||||
:key="typeKey"
|
||||
class="kb-type-card"
|
||||
:class="{ active: newDatabase.kb_type === typeKey }"
|
||||
@click="handleKbTypeChange(typeKey)"
|
||||
>
|
||||
<div class="card-header">
|
||||
<component :is="getKbTypeIcon(typeKey)" class="type-icon" />
|
||||
<span class="type-title">{{ getKbTypeLabel(typeKey) }}</span>
|
||||
</div>
|
||||
</a-select-option>
|
||||
</a-select>
|
||||
<div class="card-description">{{ typeInfo.description }}</div>
|
||||
<div class="card-features">
|
||||
<span class="feature-tag">{{ getKbTypeFeature(typeKey) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 类型说明 -->
|
||||
<div class="kb-type-guide" v-if="newDatabase.kb_type">
|
||||
<!-- <div class="kb-type-guide" v-if="newDatabase.kb_type">
|
||||
<a-alert
|
||||
:message="getKbTypeDescription(newDatabase.kb_type)"
|
||||
:type="getKbTypeAlertType(newDatabase.kb_type)"
|
||||
show-icon
|
||||
style="margin: 12px 0;"
|
||||
/>
|
||||
</div>
|
||||
</div> -->
|
||||
|
||||
<h3>知识库名称<span style="color: var(--error-color)">*</span></h3>
|
||||
<a-input v-model:value="newDatabase.name" placeholder="新建知识库名称" size="large" />
|
||||
|
||||
<h3>嵌入模型</h3>
|
||||
<a-select v-model:value="newDatabase.embed_model_name" :options="embedModelOptions" style="width: 100%;" size="large" />
|
||||
|
||||
<!-- 根据类型显示不同配置 -->
|
||||
<div v-if="newDatabase.kb_type === 'chroma' || newDatabase.kb_type === 'milvus'" class="chunk-config">
|
||||
<h3>分块配置</h3>
|
||||
<div class="chunk-params">
|
||||
<div class="param-row">
|
||||
<label>分块大小:</label>
|
||||
<a-input-number
|
||||
v-model:value="newDatabase.chunk_size"
|
||||
:min="100"
|
||||
:max="5000"
|
||||
:step="100"
|
||||
style="width: 120px;"
|
||||
/>
|
||||
<span class="param-hint">每个文本片段的最大字符数(100-5000)</span>
|
||||
</div>
|
||||
<div class="param-row">
|
||||
<label>重叠长度:</label>
|
||||
<a-input-number
|
||||
v-model:value="newDatabase.chunk_overlap"
|
||||
:min="0"
|
||||
:max="500"
|
||||
:step="50"
|
||||
style="width: 120px;"
|
||||
/>
|
||||
<span class="param-hint">相邻片段间的重叠字符数(0-500)</span>
|
||||
</div>
|
||||
<!-- <div v-if="newDatabase.kb_type === 'chroma' || newDatabase.kb_type === 'milvus'" class="storage-config">
|
||||
<h3>存储配置</h3>
|
||||
<div class="param-row">
|
||||
<label>存储方式:</label>
|
||||
<a-select v-model:value="newDatabase.storage" style="width: 200px;">
|
||||
<a-select-option value="DemoA">DemoA</a-select-option>
|
||||
<a-select-option value="DemoB">DemoB</a-select-option>
|
||||
</a-select>
|
||||
<span class="param-hint">存储方式配置(功能预留)</span>
|
||||
</div>
|
||||
</div>
|
||||
</div> -->
|
||||
|
||||
<h3 style="margin-top: 20px;">知识库描述</h3>
|
||||
<p style="color: var(--gray-700); font-size: 14px;">在智能体流程中,这里的描述会作为工具的描述。智能体会根据知识库的标题和描述来选择合适的工具。所以这里描述的越详细,智能体越容易选择到合适的工具。</p>
|
||||
@ -101,12 +93,18 @@
|
||||
</div>
|
||||
<div class="info">
|
||||
<h3>{{ database.name }}</h3>
|
||||
<p><span>{{ database.files ? Object.keys(database.files).length : 0 }} 文件</span></p>
|
||||
<p>
|
||||
<span>{{ database.files ? Object.keys(database.files).length : 0 }} 文件</span>
|
||||
<span class="created-time-inline" v-if="database.created_at">
|
||||
• {{ formatCreatedTime(database.created_at) }}
|
||||
</span>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<a-tooltip :title="database.description || '暂无描述'">
|
||||
<!-- <a-tooltip :title="database.description || '暂无描述'">
|
||||
<p class="description">{{ database.description || '暂无描述' }}</p>
|
||||
</a-tooltip>
|
||||
</a-tooltip> -->
|
||||
<p class="description">{{ database.description || '暂无描述' }}</p>
|
||||
<div class="tags">
|
||||
<a-tag color="blue" v-if="database.embed_info?.name">{{ database.embed_info.name }}</a-tag>
|
||||
<a-tag color="green" v-if="database.embed_info?.dimension">{{ database.embed_info.dimension }}</a-tag>
|
||||
@ -118,6 +116,7 @@
|
||||
{{ getKbTypeLabel(database.kb_type || 'lightrag') }}
|
||||
</a-tag>
|
||||
</div>
|
||||
|
||||
<!-- <button @click="deleteDatabase(database.collection_name)">删除</button> -->
|
||||
</div>
|
||||
</div>
|
||||
@ -129,8 +128,7 @@ import { ref, onMounted, reactive, watch, computed } from 'vue'
|
||||
import { useRouter, useRoute } from 'vue-router';
|
||||
import { useConfigStore } from '@/stores/config';
|
||||
import { message } from 'ant-design-vue'
|
||||
import { ReadFilled, DatabaseOutlined, ThunderboltOutlined } from '@ant-design/icons-vue'
|
||||
import { BookPlus, Database, Zap } from 'lucide-vue-next';
|
||||
import { BookPlus, Database, Zap, FileDigit, Waypoints, Building2 } from 'lucide-vue-next';
|
||||
import { databaseApi, typeApi } from '@/apis/knowledge_api';
|
||||
import HeaderComponent from '@/components/HeaderComponent.vue';
|
||||
|
||||
@ -156,10 +154,9 @@ const emptyEmbedInfo = {
|
||||
name: '',
|
||||
description: '',
|
||||
embed_model_name: configStore.config?.embed_model,
|
||||
kb_type: 'lightrag', // 默认为 LightRAG
|
||||
kb_type: 'chroma', // 默认为 Milvus
|
||||
// Vector 知识库特有配置
|
||||
chunk_size: 1000,
|
||||
chunk_overlap: 200,
|
||||
storage: '', // 存储方式配置
|
||||
}
|
||||
|
||||
const newDatabase = reactive({
|
||||
@ -193,7 +190,12 @@ const loadDatabases = () => {
|
||||
databaseApi.getDatabases()
|
||||
.then(data => {
|
||||
console.log(data)
|
||||
databases.value = data.databases
|
||||
// 按照创建时间排序,最新的在前面
|
||||
databases.value = data.databases.sort((a, b) => {
|
||||
const timeA = a.created_at ? new Date(a.created_at).getTime() : 0
|
||||
const timeB = b.created_at ? new Date(b.created_at).getTime() : 0
|
||||
return timeB - timeA // 降序排列,最新的在前面
|
||||
})
|
||||
state.loading = false
|
||||
})
|
||||
.catch(error => {
|
||||
@ -225,21 +227,21 @@ const getKbTypeLabel = (type) => {
|
||||
|
||||
const getKbTypeIcon = (type) => {
|
||||
const icons = {
|
||||
lightrag: Database,
|
||||
chroma: Zap,
|
||||
milvus: ThunderboltOutlined
|
||||
lightrag: Waypoints,
|
||||
chroma: FileDigit,
|
||||
milvus: Building2
|
||||
}
|
||||
return icons[type] || Database
|
||||
}
|
||||
|
||||
const getKbTypeDescription = (type) => {
|
||||
const descriptions = {
|
||||
lightrag: '🔥 图结构索引 • 智能查询 • 关系挖掘 • 复杂推理',
|
||||
chroma: '⚡ 轻量向量 • 快速开发 • 本地部署 • 简单易用',
|
||||
milvus: '🚀 生产级 • 高性能 • 分布式 • 企业级部署'
|
||||
}
|
||||
return descriptions[type] || ''
|
||||
}
|
||||
// const getKbTypeDescription = (type) => {
|
||||
// const descriptions = {
|
||||
// lightrag: '🔥 图结构索引 • 智能查询 • 关系挖掘 • 复杂推理',
|
||||
// chroma: '⚡ 轻量向量 • 快速开发 • 本地部署 • 简单易用',
|
||||
// milvus: '🚀 生产级 • 高性能 • 分布式 • 企业级部署'
|
||||
// }
|
||||
// return descriptions[type] || ''
|
||||
// }
|
||||
|
||||
const getKbTypeAlertType = (type) => {
|
||||
const types = {
|
||||
@ -259,14 +261,47 @@ const getKbTypeColor = (type) => {
|
||||
return colors[type] || 'blue'
|
||||
}
|
||||
|
||||
const getKbTypeFeature = (type) => {
|
||||
const features = {
|
||||
lightrag: '图结构索引',
|
||||
chroma: '轻量向量',
|
||||
milvus: '生产级部署'
|
||||
}
|
||||
return features[type] || ''
|
||||
}
|
||||
|
||||
// 格式化创建时间
|
||||
const formatCreatedTime = (createdAt) => {
|
||||
if (!createdAt) return ''
|
||||
|
||||
const now = new Date()
|
||||
const createdTime = new Date(createdAt)
|
||||
const diffInMs = now.getTime() - createdTime.getTime()
|
||||
const diffInDays = Math.floor(diffInMs / (1000 * 60 * 60 * 24))
|
||||
|
||||
if (diffInDays === 0) {
|
||||
return '今天创建'
|
||||
} else if (diffInDays === 1) {
|
||||
return '昨天创建'
|
||||
} else if (diffInDays < 7) {
|
||||
return `${diffInDays} 天前创建`
|
||||
} else if (diffInDays < 30) {
|
||||
const weeks = Math.floor(diffInDays / 7)
|
||||
return `${weeks} 周前创建`
|
||||
} else if (diffInDays < 365) {
|
||||
const months = Math.floor(diffInDays / 30)
|
||||
return `${months} 个月前创建`
|
||||
} else {
|
||||
const years = Math.floor(diffInDays / 365)
|
||||
return `${years} 年前创建`
|
||||
}
|
||||
}
|
||||
|
||||
// 处理知识库类型改变
|
||||
const handleKbTypeChange = (type) => {
|
||||
console.log('知识库类型改变:', type)
|
||||
// 可以在这里根据类型设置默认值
|
||||
if (type === 'chroma' || type === 'milvus') {
|
||||
newDatabase.chunk_size = 1000
|
||||
newDatabase.chunk_overlap = 200
|
||||
}
|
||||
resetNewDatabase()
|
||||
newDatabase.kb_type = type
|
||||
}
|
||||
|
||||
const createDatabase = () => {
|
||||
@ -280,18 +315,6 @@ const createDatabase = () => {
|
||||
return
|
||||
}
|
||||
|
||||
// 向量类型的额外验证(Chroma 和 Milvus)
|
||||
if (newDatabase.kb_type === 'chroma' || newDatabase.kb_type === 'milvus') {
|
||||
if (!newDatabase.chunk_size || newDatabase.chunk_size < 100) {
|
||||
message.error('分块大小不能小于100')
|
||||
return
|
||||
}
|
||||
if (newDatabase.chunk_overlap < 0) {
|
||||
message.error('重叠长度不能小于0')
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
state.creating = true
|
||||
|
||||
const requestData = {
|
||||
@ -299,14 +322,12 @@ const createDatabase = () => {
|
||||
description: newDatabase.description?.trim() || '',
|
||||
embed_model_name: newDatabase.embed_model_name || configStore.config.embed_model,
|
||||
kb_type: newDatabase.kb_type,
|
||||
additional_params: {}
|
||||
}
|
||||
|
||||
// 添加类型特有的配置
|
||||
if (newDatabase.kb_type === 'chroma' || newDatabase.kb_type === 'milvus') {
|
||||
requestData.extra_config = {
|
||||
chunk_size: newDatabase.chunk_size,
|
||||
chunk_overlap: newDatabase.chunk_overlap,
|
||||
}
|
||||
requestData.additional_params.storage = newDatabase.storage || 'DemoA'
|
||||
}
|
||||
|
||||
databaseApi.createDatabase(requestData)
|
||||
@ -349,6 +370,150 @@ onMounted(() => {
|
||||
margin: 12px 0;
|
||||
}
|
||||
|
||||
.kb-type-cards {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, 1fr);
|
||||
gap: 16px;
|
||||
margin: 16px 0;
|
||||
|
||||
@media (max-width: 768px) {
|
||||
grid-template-columns: 1fr;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.kb-type-card {
|
||||
border: 2px solid #f0f0f0;
|
||||
border-radius: 12px;
|
||||
padding: 20px;
|
||||
cursor: pointer;
|
||||
transition: all 0.3s ease;
|
||||
background: white;
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
|
||||
&:hover {
|
||||
border-color: var(--main-color);
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
// 为不同知识库类型设置不同的悬停颜色
|
||||
&:nth-child(1):hover {
|
||||
border-color: #d3adf7;
|
||||
}
|
||||
|
||||
&:nth-child(2):hover {
|
||||
border-color: #ffd591;
|
||||
}
|
||||
|
||||
&:nth-child(3):hover {
|
||||
border-color: #ffadd2;
|
||||
}
|
||||
|
||||
&.active {
|
||||
border-color: var(--main-color);
|
||||
background: #f8faff;
|
||||
|
||||
.type-icon {
|
||||
color: var(--main-color);
|
||||
}
|
||||
|
||||
.feature-tag {
|
||||
background: rgba(24, 144, 255, 0.1);
|
||||
color: var(--main-color);
|
||||
}
|
||||
}
|
||||
|
||||
// 为不同知识库类型设置不同的主题色
|
||||
&:nth-child(1) {
|
||||
&.active {
|
||||
border-color: #d3adf7;
|
||||
background: #f9f0ff;
|
||||
|
||||
.type-icon {
|
||||
color: #722ed1;
|
||||
}
|
||||
|
||||
.feature-tag {
|
||||
background: rgba(114, 46, 209, 0.1);
|
||||
color: #722ed1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
&:nth-child(2) {
|
||||
&.active {
|
||||
border-color: #ffd591;
|
||||
background: #fff7e6;
|
||||
|
||||
.type-icon {
|
||||
color: #fa8c16;
|
||||
}
|
||||
|
||||
.feature-tag {
|
||||
background: rgba(250, 140, 22, 0.1);
|
||||
color: #fa8c16;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
&:nth-child(3) {
|
||||
&.active {
|
||||
border-color: #ffadd2;
|
||||
background: #fff1f0;
|
||||
|
||||
.type-icon {
|
||||
color: #f5222d;
|
||||
}
|
||||
|
||||
.feature-tag {
|
||||
background: rgba(245, 34, 45, 0.1);
|
||||
color: #f5222d;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.card-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
margin-bottom: 12px;
|
||||
|
||||
.type-icon {
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
color: var(--main-color);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.type-title {
|
||||
font-size: 16px;
|
||||
font-weight: 600;
|
||||
color: var(--gray-800);
|
||||
}
|
||||
}
|
||||
|
||||
.card-description {
|
||||
font-size: 13px;
|
||||
color: var(--gray-600);
|
||||
line-height: 1.5;
|
||||
margin-bottom: 12px;
|
||||
min-height: 40px;
|
||||
}
|
||||
|
||||
.card-features {
|
||||
.feature-tag {
|
||||
display: inline-block;
|
||||
padding: 4px 8px;
|
||||
background: rgba(24, 144, 255, 0.1);
|
||||
color: var(--main-color);
|
||||
border-radius: 6px;
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.chunk-config {
|
||||
margin-top: 16px;
|
||||
padding: 12px 16px;
|
||||
@ -440,6 +605,8 @@ onMounted(() => {
|
||||
height: 160px;
|
||||
padding: 20px;
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
|
||||
.top {
|
||||
display: flex;
|
||||
@ -475,6 +642,15 @@ onMounted(() => {
|
||||
p {
|
||||
color: var(--gray-900);
|
||||
font-size: small;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
flex-wrap: wrap;
|
||||
|
||||
.created-time-inline {
|
||||
color: var(--gray-500);
|
||||
font-size: 12px;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -489,6 +665,8 @@ onMounted(() => {
|
||||
text-overflow: ellipsis;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
.database-empty {
|
||||
@ -511,39 +689,4 @@ onMounted(() => {
|
||||
}
|
||||
</style>
|
||||
|
||||
<!-- 为了解决ant-select下拉选项中图标和文本对齐问题,需要使用非scoped样式 -->
|
||||
<style lang="less">
|
||||
/* 知识库类型选项样式 */
|
||||
.kb-type-option {
|
||||
.type-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
|
||||
.type-icon {
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
flex-shrink: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.type-title {
|
||||
font-weight: 500;
|
||||
}
|
||||
}
|
||||
|
||||
.type-desc {
|
||||
font-size: 12px;
|
||||
color: #666;
|
||||
margin-left: 24px;
|
||||
margin-top: 2px;
|
||||
}
|
||||
}
|
||||
|
||||
/* 确保选中项也正确对齐 */
|
||||
.ant-select-selection-item .kb-type-option .type-header {
|
||||
align-items: center;
|
||||
}
|
||||
</style>
|
||||
|
||||
@ -252,7 +252,7 @@ onMounted(async () => {
|
||||
font-size: 1.2rem;
|
||||
font-weight: 600;
|
||||
color: white;
|
||||
background: linear-gradient(135deg, var(--main-500), var(--main-600));
|
||||
background: linear-gradient(135deg, var(--main-500), var(--main-color));
|
||||
border: none;
|
||||
border-radius: 3rem;
|
||||
cursor: pointer;
|
||||
@ -262,7 +262,7 @@ onMounted(async () => {
|
||||
&:hover {
|
||||
transform: translateY(-3px);
|
||||
box-shadow: 0 7px 20px rgba(0, 0, 0, 0.15);
|
||||
background: linear-gradient(135deg, var(--main-600), var(--main-700));
|
||||
background: linear-gradient(135deg, var(--main-color), var(--main-700));
|
||||
}
|
||||
|
||||
&:active {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user