refactor(knowledge): 重构文件处理逻辑并提取到独立模块

将文件处理和URL处理逻辑从KnowledgeBase类提取到独立的indexing模块
更新模型配置和README，添加新的API Key获取链接
优化日志输出格式并添加新的文件预处理脚本

README.md

@@ -53,6 +53,7 @@ https://github.com/user-attachments/assets/15f7f315-003d-4e41-a260-739c2529f824
    SILICONFLOW_API_KEY=sk-270ea********8bfa97.e3XOMd****Q1Sk
    ```
    > 💡 [免费获取 SiliconFlow API Key](https://cloud.siliconflow.cn/i/Eo5yTHGJ)（注册即送 14 元额度）
+   > 💡 [免费获取 智谱 API Key](https://www.bigmodel.cn/invite?icode=6BruxYJDdROL5pQX%2FOeYvA%3D%3D)（注册即送 2000 万Tokens）
 
 3. **启动服务**
    ```bash

docs/changelog/v0.2.md

@@ -21,3 +21,4 @@
 - [ ] 添加用户日志与用户反馈模块，可以在 AgentView 中查看信息（🌟🌟）
 - [ ] 对话页面支持文档/图片临时上传（🌟🌟🌟🌟）
 - [ ] 在Prompt中添加对于参考文献的要求，同时要求使用链接的形式，在聊天页面点击该链接后，可以选择跳转或者弹出信息预览框（最好可以提供源文件下载的功能），这里需要将DatabaseInfoView 里面的那个文件详情的弹窗给组件化。但是同时也要能够配置，选择显示源文件还是chunk的结果，如果可以的话，最好是在一个文件里面显示所有的chunk，而不是每个chunk都单独显示。不过似乎有点难度，毕竟是有 overlap 的存在，还是分开吧。对于URL链接的这种，可以选择直接跳转就可以。
+- [ ] 各种结果的可视化：知识库检索页面，工具调用检索页面，知识图谱检索页面

scripts/preprocessors/split_data_to_subfiles.py

@@ -0,0 +1,74 @@
+import typer
+import pandas as pd
+import json
+import random
+import re
+from pathlib import Path
+
+app = typer.Typer()
+
+def sanitize_filename(name: str) -> str:
+    return re.sub(r'[\\/*?:"<>|]', "_", str(name).strip())
+
+def random_suffix() -> str:
+    return f"_{random.randint(10000000, 99999999)}"
+
+def read_table(file_path: Path) -> pd.DataFrame:
+    suffix = file_path.suffix.lower()
+    if suffix in ['.xlsx', '.xls']:
+        return pd.read_excel(file_path)
+    elif suffix == '.csv':
+        return pd.read_csv(file_path)
+    elif suffix == '.json':
+        with open(file_path, encoding='utf-8') as f:
+            data = json.load(f)
+        if isinstance(data, list) and len(data) > 1 and isinstance(data[0], dict):
+            return pd.DataFrame(data)
+        else:
+            raise ValueError("JSON 文件格式不符合要求：应为元素个数 > 1 的数组，每个元素是对象。")
+    else:
+        raise ValueError(f"不支持的文件格式：{suffix}")
+
+def export_txts(df: pd.DataFrame, output_dir: Path, title_field: str = "标题"):
+    output_dir.mkdir(parents=True, exist_ok=True)
+    df.columns = [c.strip() for c in df.columns]
+
+    if title_field not in df.columns:
+        title_field = df.columns[0]  # fallback
+
+    for idx, row in df.iterrows():
+        title = str(row.get(title_field, "")).strip()
+        if not title:
+            title = f"{str(row[df.columns[0]])}{random_suffix()}"
+        else:
+            title = sanitize_filename(title)
+
+        filename = f"{title}.txt"
+        file_path = output_dir / filename
+
+        # 构造内容：字段: 值，每行一个
+        content = "\n".join(f"{col}: {row[col]}" for col in df.columns)
+
+        with open(file_path, "w", encoding="utf-8") as f:
+            f.write(content)
+
+    typer.echo(f"✅ 成功导出 {len(df)} 个文件到目录：{output_dir}")
+
+@app.command()
+def convert(
+    input_file: Path = typer.Argument(..., help="输入文件（.xlsx/.xls/.csv/.json）"),
+    out_dir: Path = typer.Option("output", help="输出目录"),
+    title_field: str = typer.Option("标题", help="标题字段名（用于文件名）")
+):
+    """
+    将结构化数据文件（Excel/CSV/JSON）转换为多个 .txt 文件。
+    """
+    try:
+        df = read_table(input_file)
+        export_txts(df, out_dir, title_field)
+    except Exception as e:
+        typer.echo(f"❌ 错误：{e}", err=True)
+        raise typer.Exit(code=1)
+
+if __name__ == "__main__":
+    app()

server/routers/knowledge_router.py

@@ -3,8 +3,9 @@ import asyncio
 import traceback
 from fastapi import APIRouter, File, UploadFile, HTTPException, Depends, Body, Form, Query
 
-from src.utils import logger, hashstr
 from src import executor, config, knowledge_base
+from src.utils import logger, hashstr
+from src.knowledge.indexing import process_file_to_markdown
 from server.utils.auth_middleware import get_admin_user
 from server.models.user_model import User
 
@@ -373,6 +374,20 @@ async def upload_file(
 
     return {"message": "File successfully uploaded", "file_path": file_path, "db_id": db_id}
 
+@knowledge.post("/files/markdown")
+async def mark_it_down(
+    file: UploadFile = File(...),
+    current_user: User = Depends(get_admin_user)
+):
+    """调用 src.knowledge.indexing 下面的 process_file_to_markdown 解析为 markdown，参数是文件，需要管理员权限"""
+    try:
+        content = await file.read()
+        markdown_content = await process_file_to_markdown(content)
+        return {"markdown_content": markdown_content, "message": "success"}
+    except Exception as e:
+        logger.error(f"文件解析失败 {e}, {traceback.format_exc()}")
+        return {"message": f"文件解析失败 {e}", "markdown_content": ""}
+
 # =============================================================================
 # === 知识库类型分组 ===
 # =============================================================================

src/agents/chatbot/configuration.py

@@ -25,7 +25,7 @@ class ChatbotConfiguration(Configuration):
     )
 
     model: str = field(
-        default="zhipu/glm-4-plus",
+        default="siliconflow/Qwen/Qwen3-235B-A22B-Instruct-2507",
         metadata={
             "name": "智能体模型",
             "options": [],

src/knowledge/chroma_kb.py

@@ -11,6 +11,7 @@ from chromadb.config import Settings
 from chromadb.api.types import EmbeddingFunction, Documents, Embeddings
 from chromadb.utils.embedding_functions import OpenAIEmbeddingFunction
 
+from src.knowledge.indexing import process_url_to_markdown, process_file_to_markdown
 from src.knowledge.knowledge_base import KnowledgeBase
 from src.knowledge.kb_utils import split_text_into_chunks, split_text_into_qa_chunks, prepare_item_metadata, get_embedding_config
 from src.utils import logger, hashstr
@@ -188,9 +189,9 @@ class ChromaKB(KnowledgeBase):
             try:
                 # 根据内容类型处理内容
                 if content_type == "file":
-                    markdown_content = await self._process_file_to_markdown(item, params=params)
+                    markdown_content = await process_file_to_markdown(item, params=params)
                 else:  # URL
-                    markdown_content = await self._process_url_to_markdown(item, params=params)
+                    markdown_content = await process_url_to_markdown(item, params=params)
 
                 # 分割文本成块
                 chunks = self._split_text_into_chunks(markdown_content, file_id, filename, params)

src/knowledge/indexing.py

@@ -216,3 +216,120 @@ async def parse_pdf_async(file, params=None):
 
 async def parse_image_async(file, params=None):
     return await asyncio.to_thread(parse_image, file, params=params)
+
+async def process_file_to_markdown(file_path: str, params: dict | None = None) -> str:
+    """
+    将不同类型的文件转换为markdown格式
+
+    Args:
+        file_path: 文件路径
+        params: 处理参数
+
+    Returns:
+        markdown格式内容
+    """
+    file_path_obj = Path(file_path)
+    file_ext = file_path_obj.suffix.lower()
+
+    if file_ext == '.pdf':
+        # 使用 OCR 处理 PDF
+        text = await parse_pdf_async(str(file_path_obj), params=params)
+        return f"# {file_path_obj.name}\n\n{text}"
+
+    elif file_ext in ['.txt', '.md']:
+        # 直接读取文本文件
+        with open(file_path_obj, encoding='utf-8') as f:
+            content = f.read()
+        return f"# {file_path_obj.name}\n\n{content}"
+
+    elif file_ext in ['.doc', '.docx']:
+        # 处理 Word 文档
+        from docx import Document  # type: ignore
+        doc = Document(file_path_obj)
+        text = '\n'.join([para.text for para in doc.paragraphs])
+        return f"# {file_path_obj.name}\n\n{text}"
+
+    elif file_ext in ['.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif']:
+        # 使用 OCR 处理图片
+        text = await parse_image_async(str(file_path_obj), params=params)
+        return f"# {file_path_obj.name}\n\n{text}"
+
+    elif file_ext in ['.html', '.htm']:
+        # 使用 BeautifulSoup 处理 HTML 文件
+        from markdownify import markdownify as md
+        with open(file_path_obj, encoding='utf-8') as f:
+            content = f.read()
+        text = md(content, heading_style="ATX")
+        return f"# {file_path_obj.name}\n\n{text}"
+
+    elif file_ext == '.csv':
+        # 处理 CSV 文件
+        import pandas as pd
+        df = pd.read_csv(file_path_obj)
+        # 将每一行数据与表头组合成独立的表格
+        markdown_content = f"# {file_path_obj.name}\n\n"
+
+        for index, row in df.iterrows():
+            # 创建包含表头和当前行的小表格
+            row_df = pd.DataFrame([row], columns=df.columns)
+            markdown_table = row_df.to_markdown(index=False)
+            markdown_content += f"{markdown_table}\n\n"
+
+        return markdown_content.strip()
+
+    elif file_ext in ['.xls', '.xlsx']:
+        # 处理 Excel 文件
+        import pandas as pd
+        # 读取所有工作表
+        excel_file = pd.ExcelFile(file_path_obj)
+        markdown_content = f"# {file_path_obj.name}\n\n"
+
+        for sheet_name in excel_file.sheet_names:
+            df = pd.read_excel(file_path_obj, sheet_name=sheet_name)
+            markdown_content += f"## {sheet_name}\n\n"
+
+            # 将每一行数据与表头组合成独立的表格
+            for index, row in df.iterrows():
+                # 创建包含表头和当前行的小表格
+                row_df = pd.DataFrame([row], columns=df.columns)
+                markdown_table = row_df.to_markdown(index=False)
+                markdown_content += f"{markdown_table}\n\n"
+
+        return markdown_content.strip()
+
+    elif file_ext == '.json':
+        # 处理 JSON 文件
+        import json
+        with open(file_path_obj, encoding='utf-8') as f:
+            data = json.load(f)
+        # 将 JSON 数据格式化为 markdown 代码块
+        json_str = json.dumps(data, ensure_ascii=False, indent=2)
+        return f"# {file_path_obj.name}\n\n```json\n{json_str}\n```"
+
+    else:
+        # 尝试作为文本文件读取
+        raise ValueError(f"Unsupported file type: {file_ext}")
+
+async def process_url_to_markdown(url: str, params: dict | None = None) -> str:
+    """
+    将URL转换为markdown格式
+
+    Args:
+        url: URL地址
+        params: 处理参数
+
+    Returns:
+        markdown格式内容
+    """
+    import requests
+    from bs4 import BeautifulSoup
+
+    try:
+        response = requests.get(url, timeout=30)
+        soup = BeautifulSoup(response.content, 'html.parser')
+        text_content = soup.get_text()
+        return f"# {url}\n\n{text_content}"
+    except Exception as e:
+        logger.error(f"Failed to process URL {url}: {e}")
+        return f"# {url}\n\nFailed to process URL: {e}"
+

src/knowledge/knowledge_base.py

@@ -377,122 +377,3 @@ class KnowledgeBase(ABC):
                 json.dump(data, f, ensure_ascii=False, indent=2)
         except Exception as e:
             logger.error(f"Failed to save {self.kb_type} metadata: {e}")
-
-    async def _process_file_to_markdown(self, file_path: str, params: dict | None = None) -> str:
-        """
-        将不同类型的文件转换为markdown格式
-
-        Args:
-            file_path: 文件路径
-            params: 处理参数
-
-        Returns:
-            markdown格式内容
-        """
-        file_path_obj = Path(file_path)
-        file_ext = file_path_obj.suffix.lower()
-
-        if file_ext == '.pdf':
-            # 使用 OCR 处理 PDF
-            from src.knowledge.indexing import parse_pdf_async
-            text = await parse_pdf_async(str(file_path_obj), params=params)
-            return f"# {file_path_obj.name}\n\n{text}"
-
-        elif file_ext in ['.txt', '.md']:
-            # 直接读取文本文件
-            with open(file_path_obj, encoding='utf-8') as f:
-                content = f.read()
-            return f"# {file_path_obj.name}\n\n{content}"
-
-        elif file_ext in ['.doc', '.docx']:
-            # 处理 Word 文档
-            from docx import Document  # type: ignore
-            doc = Document(file_path_obj)
-            text = '\n'.join([para.text for para in doc.paragraphs])
-            return f"# {file_path_obj.name}\n\n{text}"
-
-        elif file_ext in ['.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif']:
-            # 使用 OCR 处理图片
-            from src.knowledge.indexing import parse_image_async
-            text = await parse_image_async(str(file_path_obj), params=params)
-            return f"# {file_path_obj.name}\n\n{text}"
-
-        elif file_ext in ['.html', '.htm']:
-            # 使用 BeautifulSoup 处理 HTML 文件
-            from markdownify import markdownify as md
-            with open(file_path_obj, encoding='utf-8') as f:
-                content = f.read()
-            text = md(content, heading_style="ATX")
-            return f"# {file_path_obj.name}\n\n{text}"
-
-        elif file_ext == '.csv':
-            # 处理 CSV 文件
-            import pandas as pd
-            df = pd.read_csv(file_path_obj)
-            # 将每一行数据与表头组合成独立的表格
-            markdown_content = f"# {file_path_obj.name}\n\n"
-
-            for index, row in df.iterrows():
-                # 创建包含表头和当前行的小表格
-                row_df = pd.DataFrame([row], columns=df.columns)
-                markdown_table = row_df.to_markdown(index=False)
-                markdown_content += f"{markdown_table}\n\n"
-
-            return markdown_content.strip()
-
-        elif file_ext in ['.xls', '.xlsx']:
-            # 处理 Excel 文件
-            import pandas as pd
-            # 读取所有工作表
-            excel_file = pd.ExcelFile(file_path_obj)
-            markdown_content = f"# {file_path_obj.name}\n\n"
-
-            for sheet_name in excel_file.sheet_names:
-                df = pd.read_excel(file_path_obj, sheet_name=sheet_name)
-                markdown_content += f"## {sheet_name}\n\n"
-
-                # 将每一行数据与表头组合成独立的表格
-                for index, row in df.iterrows():
-                    # 创建包含表头和当前行的小表格
-                    row_df = pd.DataFrame([row], columns=df.columns)
-                    markdown_table = row_df.to_markdown(index=False)
-                    markdown_content += f"{markdown_table}\n\n"
-
-            return markdown_content.strip()
-
-        elif file_ext == '.json':
-            # 处理 JSON 文件
-            import json
-            with open(file_path_obj, encoding='utf-8') as f:
-                data = json.load(f)
-            # 将 JSON 数据格式化为 markdown 代码块
-            json_str = json.dumps(data, ensure_ascii=False, indent=2)
-            return f"# {file_path_obj.name}\n\n```json\n{json_str}\n```"
-
-        else:
-            # 尝试作为文本文件读取
-            raise ValueError(f"Unsupported file type: {file_ext}")
-
-    async def _process_url_to_markdown(self, url: str,
-                                     params: dict | None = None) -> str:
-        """
-        将URL转换为markdown格式
-
-        Args:
-            url: URL地址
-            params: 处理参数
-
-        Returns:
-            markdown格式内容
-        """
-        import requests
-        from bs4 import BeautifulSoup
-
-        try:
-            response = requests.get(url, timeout=30)
-            soup = BeautifulSoup(response.content, 'html.parser')
-            text_content = soup.get_text()
-            return f"# {url}\n\n{text_content}"
-        except Exception as e:
-            logger.error(f"Failed to process URL {url}: {e}")
-            return f"# {url}\n\nFailed to process URL: {e}"

src/knowledge/lightrag_kb.py

@@ -10,6 +10,7 @@ from lightrag.utils import EmbeddingFunc, setup_logger
 from lightrag.kg.shared_storage import initialize_pipeline_status
 
 from src.knowledge.knowledge_base import KnowledgeBase
+from src.knowledge.indexing import process_url_to_markdown, process_file_to_markdown
 from src.knowledge.kb_utils import prepare_item_metadata, get_embedding_config
 from src import config
 from src.utils import logger, hashstr, get_docker_safe_url
@@ -166,11 +167,11 @@ class LightRagKB(KnowledgeBase):
             try:
                 # 根据内容类型处理内容
                 if content_type == "file":
-                    markdown_content = await self._process_file_to_markdown(item, params=params)
+                    markdown_content = await process_file_to_markdown(item, params=params)
                     markdown_content_lines = markdown_content[:100].replace('\n', ' ')
                     logger.info(f"Markdown content: {markdown_content_lines}...")
                 else:  # URL
-                    markdown_content = await self._process_url_to_markdown(item, params=params)
+                    markdown_content = await process_url_to_markdown(item, params=params)
 
                 # 使用 LightRAG 插入内容
                 await rag.ainsert(

src/knowledge/milvus_kb.py

@@ -16,6 +16,7 @@ from pymilvus import (
 from src import config
 from src.models.embedding import OtherEmbedding
 from src.knowledge.knowledge_base import KnowledgeBase
+from src.knowledge.indexing import process_url_to_markdown, process_file_to_markdown
 from src.knowledge.kb_utils import split_text_into_chunks, split_text_into_qa_chunks, prepare_item_metadata, get_embedding_config
 from src.utils import logger, hashstr
 
@@ -258,9 +259,9 @@ class MilvusKB(KnowledgeBase):
 
             try:
                 if content_type == "file":
-                    markdown_content = await self._process_file_to_markdown(item, params=params)
+                    markdown_content = await process_file_to_markdown(item, params=params)
                 else:
-                    markdown_content = await self._process_url_to_markdown(item, params=params)
+                    markdown_content = await process_url_to_markdown(item, params=params)
 
                 chunks = self._split_text_into_chunks(markdown_content, file_id, filename, params)
                 logger.info(f"Split {filename} into {len(chunks)} chunks")

src/models/embedding.py

@@ -48,7 +48,7 @@ class BaseEmbeddingModel:
         return await asyncio.to_thread(self.batch_encode, messages, batch_size)
 
     def batch_encode(self, messages, batch_size=40):
-        logger.info(f"Batch encoding {len(messages)} messages")
+        # logger.info(f"Batch encoding {len(messages)} messages")
         data = []
 
         if len(messages) > batch_size:
@@ -61,7 +61,7 @@ class BaseEmbeddingModel:
 
         for i in range(0, len(messages), batch_size):
             group_msg = messages[i:i+batch_size]
-            logger.info(f"Encoding {i} to {i+batch_size} with {len(messages)} messages")
+            logger.info(f"Encoding [{i}/{len(messages)}] messages (bsz={batch_size})")
             response = self.encode(group_msg)
             # logger.debug(f"Response: {len(response)=}, {len(group_msg)=}, {len(response[0])=}")
             data.extend(response)

src/static/models.yaml

@@ -11,14 +11,13 @@ MODEL_NAMES:
     name: OpenAI
     url: https://platform.openai.com/docs/models
     base_url: https://api.openai.com/v1
-    default: gpt-3.5-turbo
+    default: gpt-4o-mini
     env:
       - OPENAI_API_KEY
     models:
       - gpt-4
       - gpt-4o
       - gpt-4o-mini
-      - gpt-3.5-turbo
 
   deepseek:
     name: DeepSeek
@@ -35,15 +34,13 @@ MODEL_NAMES:
     name: 智谱AI (Zhipu)
     url: https://open.bigmodel.cn/dev/api
     base_url: https://open.bigmodel.cn/api/paas/v4/
-    default: glm-4-flash
+    default: glm-4.5-flash
     env:
       - ZHIPUAI_API_KEY
     models:
-      - glm-4
+      - glm-4.5
-      - glm-4-plus
+      - glm-4.5-air
-      - glm-4-air
+      - glm-4.5-flash
-      - glm-4-flash
-      - glm-z1-air
 
   siliconflow:
     name: SiliconFlow
@@ -55,8 +52,10 @@ MODEL_NAMES:
     models:
       - Pro/deepseek-ai/DeepSeek-R1
       - Pro/deepseek-ai/DeepSeek-V3
-      - Qwen/QwQ-32B
+      - Qwen/Qwen3-235B-A22B-Thinking-2507
-      - Qwen/Qwen3-8B
+      - Qwen/Qwen3-235B-A22B-Instruct-2507
+      - moonshotai/Kimi-K2-Instruct
+      - zai-org/GLM-4.5
 
   together.ai:
     name: Together.ai
@@ -66,34 +65,33 @@ MODEL_NAMES:
     env:
       - TOGETHER_API_KEY
     models:
-      - meta-llama/Llama-3.3-70B-Instruct-Turbo
       - meta-llama/Llama-3.3-70B-Instruct-Turbo-Free
-      - deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free
-      - Qwen/QwQ-32B
 
   dashscope:
     name: 阿里百炼 (DashScope)
     url: https://bailian.console.aliyun.com/?switchAgent=10226727&productCode=p_efm#/model-market
     base_url: https://dashscope.aliyuncs.com/compatible-mode/v1
-    default: qwen3-235b-a22b
+    default: qwen-max-latest
     env:
       - DASHSCOPE_API_KEY
     models:
       - qwen-max-latest
-      - qwen3-235b-a22b
+      - qwen-plus-latest
-      - qwen3-32b
+      - qwen-turbo-latest
+      - qwen3-235b-a22b-thinking-2507
+      - qwen3-235b-a22b-instruct-2507
 
   ark:
     name: 豆包（Ark）
     url: https://console.volcengine.com/ark/region:ark+cn-beijing/model
     base_url: https://ark.cn-beijing.volces.com/api/v3
-    default: doubao-1-5-pro-32k-250115
+    default: doubao-seed-1-6-250615
     env:
       - ARK_API_KEY
     models:
-      - doubao-1-5-pro-32k-250115
+      - doubao-seed-1-6-250615
-      - doubao-1-5-lite-32k-250115
+      - doubao-seed-1-6-thinking-250715
-      - deepseek-r1-250120
+      - doubao-seed-1-6-flash-250715
 
   openrouter:
     name: OpenRouter
@@ -104,14 +102,9 @@ MODEL_NAMES:
       - OPENROUTER_API_KEY
     models:
       - openai/gpt-4o
-      - openai/gpt-4o-mini
+      - x-ai/grok-4
-      - google/gemini-2.5-pro-exp-03-25:free
+      - google/gemini-2.5-pro
-      - x-ai/grok-3-beta
+      - anthropic/claude-sonnet-4
-      - meta-llama/llama-4-maverick
-      - meta-llama/llama-4-maverick:free
-      - anthropic/claude-3.7-sonnet
-      - anthropic/claude-3.7-sonnet:thinking
-
 
 EMBED_MODEL_INFO:
   siliconflow/BAAI/bge-m3: