diff --git a/README.md b/README.md index de18e020..90d33e6a 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ https://github.com/user-attachments/assets/15f7f315-003d-4e41-a260-739c2529f824 SILICONFLOW_API_KEY=sk-270ea********8bfa97.e3XOMd****Q1Sk ``` > 💡 [免费获取 SiliconFlow API Key](https://cloud.siliconflow.cn/i/Eo5yTHGJ)（注册即送 14 元额度） + > 💡 [免费获取 智谱 API Key](https://www.bigmodel.cn/invite?icode=6BruxYJDdROL5pQX%2FOeYvA%3D%3D)（注册即送 2000 万Tokens） 3. **启动服务** ```bash diff --git a/docs/changelog/v0.2.md b/docs/changelog/v0.2.md index 3afaaf47..258cc098 100644 --- a/docs/changelog/v0.2.md +++ b/docs/changelog/v0.2.md @@ -21,3 +21,4 @@ - [ ] 添加用户日志与用户反馈模块，可以在 AgentView 中查看信息（🌟🌟） - [ ] 对话页面支持文档/图片临时上传（🌟🌟🌟🌟） - [ ] 在Prompt中添加对于参考文献的要求，同时要求使用链接的形式，在聊天页面点击该链接后，可以选择跳转或者弹出信息预览框（最好可以提供源文件下载的功能），这里需要将DatabaseInfoView 里面的那个文件详情的弹窗给组件化。但是同时也要能够配置，选择显示源文件还是chunk的结果，如果可以的话，最好是在一个文件里面显示所有的chunk，而不是每个chunk都单独显示。不过似乎有点难度，毕竟是有 overlap 的存在，还是分开吧。对于URL链接的这种，可以选择直接跳转就可以。 +- [ ] 各种结果的可视化：知识库检索页面，工具调用检索页面，知识图谱检索页面 diff --git a/scripts/preprocessors/split_data_to_subfiles.py b/scripts/preprocessors/split_data_to_subfiles.py new file mode 100644 index 00000000..f74a1072 --- /dev/null +++ b/scripts/preprocessors/split_data_to_subfiles.py @@ -0,0 +1,74 @@ +import typer +import pandas as pd +import json +import random +import re +from pathlib import Path + +app = typer.Typer() + +def sanitize_filename(name: str) -> str: + return re.sub(r'[\\/*?:"<>|]', "_", str(name).strip()) + +def random_suffix() -> str: + return f"_{random.randint(10000000, 99999999)}" + +def read_table(file_path: Path) -> pd.DataFrame: + suffix = file_path.suffix.lower() + if suffix in ['.xlsx', '.xls']: + return pd.read_excel(file_path) + elif suffix == '.csv': + return pd.read_csv(file_path) + elif suffix == '.json': + with open(file_path, encoding='utf-8') as f: + data = json.load(f) + if isinstance(data, list) and len(data) > 1 and isinstance(data[0], dict): + return pd.DataFrame(data) + else: + raise ValueError("JSON 文件格式不符合要求：应为元素个数 > 1 的数组，每个元素是对象。") + else: + raise ValueError(f"不支持的文件格式：{suffix}") + +def export_txts(df: pd.DataFrame, output_dir: Path, title_field: str = "标题"): + output_dir.mkdir(parents=True, exist_ok=True) + df.columns = [c.strip() for c in df.columns] + + if title_field not in df.columns: + title_field = df.columns[0] # fallback + + for idx, row in df.iterrows(): + title = str(row.get(title_field, "")).strip() + if not title: + title = f"{str(row[df.columns[0]])}{random_suffix()}" + else: + title = sanitize_filename(title) + + filename = f"{title}.txt" + file_path = output_dir / filename + + # 构造内容：字段: 值，每行一个 + content = "\n".join(f"{col}: {row[col]}" for col in df.columns) + + with open(file_path, "w", encoding="utf-8") as f: + f.write(content) + + typer.echo(f"✅ 成功导出 {len(df)} 个文件到目录：{output_dir}") + +@app.command() +def convert( + input_file: Path = typer.Argument(..., help="输入文件（.xlsx/.xls/.csv/.json）"), + out_dir: Path = typer.Option("output", help="输出目录"), + title_field: str = typer.Option("标题", help="标题字段名（用于文件名）") +): + """ + 将结构化数据文件（Excel/CSV/JSON）转换为多个 .txt 文件。 + """ + try: + df = read_table(input_file) + export_txts(df, out_dir, title_field) + except Exception as e: + typer.echo(f"❌ 错误：{e}", err=True) + raise typer.Exit(code=1) + +if __name__ == "__main__": + app() diff --git a/server/routers/knowledge_router.py b/server/routers/knowledge_router.py index c886d480..40628c3c 100644 --- a/server/routers/knowledge_router.py +++ b/server/routers/knowledge_router.py @@ -3,8 +3,9 @@ import asyncio import traceback from fastapi import APIRouter, File, UploadFile, HTTPException, Depends, Body, Form, Query -from src.utils import logger, hashstr from src import executor, config, knowledge_base +from src.utils import logger, hashstr +from src.knowledge.indexing import process_file_to_markdown from server.utils.auth_middleware import get_admin_user from server.models.user_model import User @@ -373,6 +374,20 @@ async def upload_file( return {"message": "File successfully uploaded", "file_path": file_path, "db_id": db_id} +@knowledge.post("/files/markdown") +async def mark_it_down( + file: UploadFile = File(...), + current_user: User = Depends(get_admin_user) +): + """调用 src.knowledge.indexing 下面的 process_file_to_markdown 解析为 markdown，参数是文件，需要管理员权限""" + try: + content = await file.read() + markdown_content = await process_file_to_markdown(content) + return {"markdown_content": markdown_content, "message": "success"} + except Exception as e: + logger.error(f"文件解析失败 {e}, {traceback.format_exc()}") + return {"message": f"文件解析失败 {e}", "markdown_content": ""} + # ============================================================================= # === 知识库类型分组 === # ============================================================================= diff --git a/src/agents/chatbot/configuration.py b/src/agents/chatbot/configuration.py index 795d4f29..8e8772c4 100644 --- a/src/agents/chatbot/configuration.py +++ b/src/agents/chatbot/configuration.py @@ -25,7 +25,7 @@ class ChatbotConfiguration(Configuration): ) model: str = field( - default="zhipu/glm-4-plus", + default="siliconflow/Qwen/Qwen3-235B-A22B-Instruct-2507", metadata={ "name": "智能体模型", "options": [], diff --git a/src/knowledge/chroma_kb.py b/src/knowledge/chroma_kb.py index ab3419d0..96d3175a 100644 --- a/src/knowledge/chroma_kb.py +++ b/src/knowledge/chroma_kb.py @@ -11,6 +11,7 @@ from chromadb.config import Settings from chromadb.api.types import EmbeddingFunction, Documents, Embeddings from chromadb.utils.embedding_functions import OpenAIEmbeddingFunction +from src.knowledge.indexing import process_url_to_markdown, process_file_to_markdown from src.knowledge.knowledge_base import KnowledgeBase from src.knowledge.kb_utils import split_text_into_chunks, split_text_into_qa_chunks, prepare_item_metadata, get_embedding_config from src.utils import logger, hashstr @@ -188,9 +189,9 @@ class ChromaKB(KnowledgeBase): try: # 根据内容类型处理内容 if content_type == "file": - markdown_content = await self._process_file_to_markdown(item, params=params) + markdown_content = await process_file_to_markdown(item, params=params) else: # URL - markdown_content = await self._process_url_to_markdown(item, params=params) + markdown_content = await process_url_to_markdown(item, params=params) # 分割文本成块 chunks = self._split_text_into_chunks(markdown_content, file_id, filename, params) diff --git a/src/knowledge/indexing.py b/src/knowledge/indexing.py index cc993a7c..72f14a7c 100644 --- a/src/knowledge/indexing.py +++ b/src/knowledge/indexing.py @@ -216,3 +216,120 @@ async def parse_pdf_async(file, params=None): async def parse_image_async(file, params=None): return await asyncio.to_thread(parse_image, file, params=params) + +async def process_file_to_markdown(file_path: str, params: dict | None = None) -> str: + """ + 将不同类型的文件转换为markdown格式 + + Args: + file_path: 文件路径 + params: 处理参数 + + Returns: + markdown格式内容 + """ + file_path_obj = Path(file_path) + file_ext = file_path_obj.suffix.lower() + + if file_ext == '.pdf': + # 使用 OCR 处理 PDF + text = await parse_pdf_async(str(file_path_obj), params=params) + return f"# {file_path_obj.name}\n\n{text}" + + elif file_ext in ['.txt', '.md']: + # 直接读取文本文件 + with open(file_path_obj, encoding='utf-8') as f: + content = f.read() + return f"# {file_path_obj.name}\n\n{content}" + + elif file_ext in ['.doc', '.docx']: + # 处理 Word 文档 + from docx import Document # type: ignore + doc = Document(file_path_obj) + text = '\n'.join([para.text for para in doc.paragraphs]) + return f"# {file_path_obj.name}\n\n{text}" + + elif file_ext in ['.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif']: + # 使用 OCR 处理图片 + text = await parse_image_async(str(file_path_obj), params=params) + return f"# {file_path_obj.name}\n\n{text}" + + elif file_ext in ['.html', '.htm']: + # 使用 BeautifulSoup 处理 HTML 文件 + from markdownify import markdownify as md + with open(file_path_obj, encoding='utf-8') as f: + content = f.read() + text = md(content, heading_style="ATX") + return f"# {file_path_obj.name}\n\n{text}" + + elif file_ext == '.csv': + # 处理 CSV 文件 + import pandas as pd + df = pd.read_csv(file_path_obj) + # 将每一行数据与表头组合成独立的表格 + markdown_content = f"# {file_path_obj.name}\n\n" + + for index, row in df.iterrows(): + # 创建包含表头和当前行的小表格 + row_df = pd.DataFrame([row], columns=df.columns) + markdown_table = row_df.to_markdown(index=False) + markdown_content += f"{markdown_table}\n\n" + + return markdown_content.strip() + + elif file_ext in ['.xls', '.xlsx']: + # 处理 Excel 文件 + import pandas as pd + # 读取所有工作表 + excel_file = pd.ExcelFile(file_path_obj) + markdown_content = f"# {file_path_obj.name}\n\n" + + for sheet_name in excel_file.sheet_names: + df = pd.read_excel(file_path_obj, sheet_name=sheet_name) + markdown_content += f"## {sheet_name}\n\n" + + # 将每一行数据与表头组合成独立的表格 + for index, row in df.iterrows(): + # 创建包含表头和当前行的小表格 + row_df = pd.DataFrame([row], columns=df.columns) + markdown_table = row_df.to_markdown(index=False) + markdown_content += f"{markdown_table}\n\n" + + return markdown_content.strip() + + elif file_ext == '.json': + # 处理 JSON 文件 + import json + with open(file_path_obj, encoding='utf-8') as f: + data = json.load(f) + # 将 JSON 数据格式化为 markdown 代码块 + json_str = json.dumps(data, ensure_ascii=False, indent=2) + return f"# {file_path_obj.name}\n\n```json\n{json_str}\n```" + + else: + # 尝试作为文本文件读取 + raise ValueError(f"Unsupported file type: {file_ext}") + +async def process_url_to_markdown(url: str, params: dict | None = None) -> str: + """ + 将URL转换为markdown格式 + + Args: + url: URL地址 + params: 处理参数 + + Returns: + markdown格式内容 + """ + import requests + from bs4 import BeautifulSoup + + try: + response = requests.get(url, timeout=30) + soup = BeautifulSoup(response.content, 'html.parser') + text_content = soup.get_text() + return f"# {url}\n\n{text_content}" + except Exception as e: + logger.error(f"Failed to process URL {url}: {e}") + return f"# {url}\n\nFailed to process URL: {e}" + diff --git a/src/knowledge/knowledge_base.py b/src/knowledge/knowledge_base.py index 6b8baf9e..af685f9d 100644 --- a/src/knowledge/knowledge_base.py +++ b/src/knowledge/knowledge_base.py @@ -377,122 +377,3 @@ class KnowledgeBase(ABC): json.dump(data, f, ensure_ascii=False, indent=2) except Exception as e: logger.error(f"Failed to save {self.kb_type} metadata: {e}") - - async def _process_file_to_markdown(self, file_path: str, params: dict | None = None) -> str: - """ - 将不同类型的文件转换为markdown格式 - - Args: - file_path: 文件路径 - params: 处理参数 - - Returns: - markdown格式内容 - """ - file_path_obj = Path(file_path) - file_ext = file_path_obj.suffix.lower() - - if file_ext == '.pdf': - # 使用 OCR 处理 PDF - from src.knowledge.indexing import parse_pdf_async - text = await parse_pdf_async(str(file_path_obj), params=params) - return f"# {file_path_obj.name}\n\n{text}" - - elif file_ext in ['.txt', '.md']: - # 直接读取文本文件 - with open(file_path_obj, encoding='utf-8') as f: - content = f.read() - return f"# {file_path_obj.name}\n\n{content}" - - elif file_ext in ['.doc', '.docx']: - # 处理 Word 文档 - from docx import Document # type: ignore - doc = Document(file_path_obj) - text = '\n'.join([para.text for para in doc.paragraphs]) - return f"# {file_path_obj.name}\n\n{text}" - - elif file_ext in ['.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif']: - # 使用 OCR 处理图片 - from src.knowledge.indexing import parse_image_async - text = await parse_image_async(str(file_path_obj), params=params) - return f"# {file_path_obj.name}\n\n{text}" - - elif file_ext in ['.html', '.htm']: - # 使用 BeautifulSoup 处理 HTML 文件 - from markdownify import markdownify as md - with open(file_path_obj, encoding='utf-8') as f: - content = f.read() - text = md(content, heading_style="ATX") - return f"# {file_path_obj.name}\n\n{text}" - - elif file_ext == '.csv': - # 处理 CSV 文件 - import pandas as pd - df = pd.read_csv(file_path_obj) - # 将每一行数据与表头组合成独立的表格 - markdown_content = f"# {file_path_obj.name}\n\n" - - for index, row in df.iterrows(): - # 创建包含表头和当前行的小表格 - row_df = pd.DataFrame([row], columns=df.columns) - markdown_table = row_df.to_markdown(index=False) - markdown_content += f"{markdown_table}\n\n" - - return markdown_content.strip() - - elif file_ext in ['.xls', '.xlsx']: - # 处理 Excel 文件 - import pandas as pd - # 读取所有工作表 - excel_file = pd.ExcelFile(file_path_obj) - markdown_content = f"# {file_path_obj.name}\n\n" - - for sheet_name in excel_file.sheet_names: - df = pd.read_excel(file_path_obj, sheet_name=sheet_name) - markdown_content += f"## {sheet_name}\n\n" - - # 将每一行数据与表头组合成独立的表格 - for index, row in df.iterrows(): - # 创建包含表头和当前行的小表格 - row_df = pd.DataFrame([row], columns=df.columns) - markdown_table = row_df.to_markdown(index=False) - markdown_content += f"{markdown_table}\n\n" - - return markdown_content.strip() - - elif file_ext == '.json': - # 处理 JSON 文件 - import json - with open(file_path_obj, encoding='utf-8') as f: - data = json.load(f) - # 将 JSON 数据格式化为 markdown 代码块 - json_str = json.dumps(data, ensure_ascii=False, indent=2) - return f"# {file_path_obj.name}\n\n```json\n{json_str}\n```" - - else: - # 尝试作为文本文件读取 - raise ValueError(f"Unsupported file type: {file_ext}") - - async def _process_url_to_markdown(self, url: str, - params: dict | None = None) -> str: - """ - 将URL转换为markdown格式 - - Args: - url: URL地址 - params: 处理参数 - - Returns: - markdown格式内容 - """ - import requests - from bs4 import BeautifulSoup - - try: - response = requests.get(url, timeout=30) - soup = BeautifulSoup(response.content, 'html.parser') - text_content = soup.get_text() - return f"# {url}\n\n{text_content}" - except Exception as e: - logger.error(f"Failed to process URL {url}: {e}") - return f"# {url}\n\nFailed to process URL: {e}" diff --git a/src/knowledge/lightrag_kb.py b/src/knowledge/lightrag_kb.py index d5454808..8ff6df73 100644 --- a/src/knowledge/lightrag_kb.py +++ b/src/knowledge/lightrag_kb.py @@ -10,6 +10,7 @@ from lightrag.utils import EmbeddingFunc, setup_logger from lightrag.kg.shared_storage import initialize_pipeline_status from src.knowledge.knowledge_base import KnowledgeBase +from src.knowledge.indexing import process_url_to_markdown, process_file_to_markdown from src.knowledge.kb_utils import prepare_item_metadata, get_embedding_config from src import config from src.utils import logger, hashstr, get_docker_safe_url @@ -166,11 +167,11 @@ class LightRagKB(KnowledgeBase): try: # 根据内容类型处理内容 if content_type == "file": - markdown_content = await self._process_file_to_markdown(item, params=params) + markdown_content = await process_file_to_markdown(item, params=params) markdown_content_lines = markdown_content[:100].replace('\n', ' ') logger.info(f"Markdown content: {markdown_content_lines}...") else: # URL - markdown_content = await self._process_url_to_markdown(item, params=params) + markdown_content = await process_url_to_markdown(item, params=params) # 使用 LightRAG 插入内容 await rag.ainsert( diff --git a/src/knowledge/milvus_kb.py b/src/knowledge/milvus_kb.py index f0a7727a..049bc0bc 100644 --- a/src/knowledge/milvus_kb.py +++ b/src/knowledge/milvus_kb.py @@ -16,6 +16,7 @@ from pymilvus import ( from src import config from src.models.embedding import OtherEmbedding from src.knowledge.knowledge_base import KnowledgeBase +from src.knowledge.indexing import process_url_to_markdown, process_file_to_markdown from src.knowledge.kb_utils import split_text_into_chunks, split_text_into_qa_chunks, prepare_item_metadata, get_embedding_config from src.utils import logger, hashstr @@ -258,9 +259,9 @@ class MilvusKB(KnowledgeBase): try: if content_type == "file": - markdown_content = await self._process_file_to_markdown(item, params=params) + markdown_content = await process_file_to_markdown(item, params=params) else: - markdown_content = await self._process_url_to_markdown(item, params=params) + markdown_content = await process_url_to_markdown(item, params=params) chunks = self._split_text_into_chunks(markdown_content, file_id, filename, params) logger.info(f"Split {filename} into {len(chunks)} chunks") diff --git a/src/models/embedding.py b/src/models/embedding.py index c9dddecf..9780239f 100644 --- a/src/models/embedding.py +++ b/src/models/embedding.py @@ -48,7 +48,7 @@ class BaseEmbeddingModel: return await asyncio.to_thread(self.batch_encode, messages, batch_size) def batch_encode(self, messages, batch_size=40): - logger.info(f"Batch encoding {len(messages)} messages") + # logger.info(f"Batch encoding {len(messages)} messages") data = [] if len(messages) > batch_size: @@ -61,7 +61,7 @@ class BaseEmbeddingModel: for i in range(0, len(messages), batch_size): group_msg = messages[i:i+batch_size] - logger.info(f"Encoding {i} to {i+batch_size} with {len(messages)} messages") + logger.info(f"Encoding [{i}/{len(messages)}] messages (bsz={batch_size})") response = self.encode(group_msg) # logger.debug(f"Response: {len(response)=}, {len(group_msg)=}, {len(response[0])=}") data.extend(response) diff --git a/src/static/models.yaml b/src/static/models.yaml index d747badf..8c8f1994 100644 --- a/src/static/models.yaml +++ b/src/static/models.yaml @@ -11,14 +11,13 @@ MODEL_NAMES: name: OpenAI url: https://platform.openai.com/docs/models base_url: https://api.openai.com/v1 - default: gpt-3.5-turbo + default: gpt-4o-mini env: - OPENAI_API_KEY models: - gpt-4 - gpt-4o - gpt-4o-mini - - gpt-3.5-turbo deepseek: name: DeepSeek @@ -35,15 +34,13 @@ MODEL_NAMES: name: 智谱AI (Zhipu) url: https://open.bigmodel.cn/dev/api base_url: https://open.bigmodel.cn/api/paas/v4/ - default: glm-4-flash + default: glm-4.5-flash env: - ZHIPUAI_API_KEY models: - - glm-4 - - glm-4-plus - - glm-4-air - - glm-4-flash - - glm-z1-air + - glm-4.5 + - glm-4.5-air + - glm-4.5-flash siliconflow: name: SiliconFlow @@ -55,8 +52,10 @@ MODEL_NAMES: models: - Pro/deepseek-ai/DeepSeek-R1 - Pro/deepseek-ai/DeepSeek-V3 - - Qwen/QwQ-32B - - Qwen/Qwen3-8B + - Qwen/Qwen3-235B-A22B-Thinking-2507 + - Qwen/Qwen3-235B-A22B-Instruct-2507 + - moonshotai/Kimi-K2-Instruct + - zai-org/GLM-4.5 together.ai: name: Together.ai @@ -66,34 +65,33 @@ MODEL_NAMES: env: - TOGETHER_API_KEY models: - - meta-llama/Llama-3.3-70B-Instruct-Turbo - meta-llama/Llama-3.3-70B-Instruct-Turbo-Free - - deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free - - Qwen/QwQ-32B dashscope: name: 阿里百炼 (DashScope) url: https://bailian.console.aliyun.com/?switchAgent=10226727&productCode=p_efm#/model-market base_url: https://dashscope.aliyuncs.com/compatible-mode/v1 - default: qwen3-235b-a22b + default: qwen-max-latest env: - DASHSCOPE_API_KEY models: - qwen-max-latest - - qwen3-235b-a22b - - qwen3-32b + - qwen-plus-latest + - qwen-turbo-latest + - qwen3-235b-a22b-thinking-2507 + - qwen3-235b-a22b-instruct-2507 ark: name: 豆包（Ark） url: https://console.volcengine.com/ark/region:ark+cn-beijing/model base_url: https://ark.cn-beijing.volces.com/api/v3 - default: doubao-1-5-pro-32k-250115 + default: doubao-seed-1-6-250615 env: - ARK_API_KEY models: - - doubao-1-5-pro-32k-250115 - - doubao-1-5-lite-32k-250115 - - deepseek-r1-250120 + - doubao-seed-1-6-250615 + - doubao-seed-1-6-thinking-250715 + - doubao-seed-1-6-flash-250715 openrouter: name: OpenRouter @@ -104,14 +102,9 @@ MODEL_NAMES: - OPENROUTER_API_KEY models: - openai/gpt-4o - - openai/gpt-4o-mini - - google/gemini-2.5-pro-exp-03-25:free - - x-ai/grok-3-beta - - meta-llama/llama-4-maverick - - meta-llama/llama-4-maverick:free - - anthropic/claude-3.7-sonnet - - anthropic/claude-3.7-sonnet:thinking - + - x-ai/grok-4 + - google/gemini-2.5-pro + - anthropic/claude-sonnet-4 EMBED_MODEL_INFO: siliconflow/BAAI/bge-m3: