From 59727d6d7eb14ef6760755853121ccff1ed906ee Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Thu, 23 Oct 2025 14:59:25 +0800 Subject: [PATCH] =?UTF-8?q?refactor:=20=E6=9B=B4=E6=96=B0=E6=96=87?= =?UTF-8?q?=E6=A1=A3=EF=BC=8C=E4=BC=98=E5=8C=96=E6=89=B9=E9=87=8F=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E5=92=8C=E8=A7=A3=E6=9E=90=E8=84=9A=E6=9C=AC=E8=AF=B4?= =?UTF-8?q?=E6=98=8E=EF=BC=9B=E7=A7=BB=E9=99=A4=E4=B8=8D=E5=86=8D=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E7=9A=84=E8=BD=AC=E6=8D=A2=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/advanced/document-processing.md | 22 +--- docs/changelog/faq.md | 1 - docs/intro/knowledge-base.md | 1 - scripts/batch_upload.py | 165 --------------------------- src/storage/minio/client.py | 2 + 5 files changed, 3 insertions(+), 188 deletions(-) diff --git a/docs/advanced/document-processing.md b/docs/advanced/document-processing.md index 27c696e8..02b14eb1 100644 --- a/docs/advanced/document-processing.md +++ b/docs/advanced/document-processing.md @@ -60,7 +60,7 @@ docker compose up paddlex --build ## 批量处理脚本 -系统提供便捷的批量处理脚本，支持文件上传和解析操作。 +系统提供便捷的批量处理脚本，用于高效批量上传文档。 ### 文件上传脚本 @@ -90,26 +90,6 @@ uv run scripts/batch_upload.py upload \ 提示：系统按“内容哈希”进行去重；同一知识库已存在相同内容的文件会被拒绝（409）。 -### 文件解析脚本 - -使用 `scripts/batch_upload.py trans` 将文件解析为 Markdown： - -```bash -# 批量解析文档 -uv run scripts/batch_upload.py trans \ - --db-id your_kb_id \ - --directory path/to/your/data \ - --output-dir path/to/output_markdown \ - --pattern "*.docx" \ - --base-url http://127.0.0.1:5050/api \ - --username your_username \ - --password your_password \ - --concurrency 4 \ - --recursive -``` - -**输出结果**: 解析后的 Markdown 文件将保存到指定输出目录。 - ### 脚本功能 - **进度跟踪**: 实时显示处理进度 diff --git a/docs/changelog/faq.md b/docs/changelog/faq.md index b3fa8787..5e0fad90 100644 --- a/docs/changelog/faq.md +++ b/docs/changelog/faq.md @@ -32,7 +32,6 @@ - 批量上传与转换示例？ - 上传入库：`uv run scripts/batch_upload.py upload --db-id --directory --username --password

--base-url http://127.0.0.1:5050/api` - - 转 Markdown：`uv run scripts/batch_upload.py trans --db-id --directory

--username --password

` - 参考：高级配置 → 文档解析 - 登录失败被锁定？ diff --git a/docs/intro/knowledge-base.md b/docs/intro/knowledge-base.md index cb1fc050..aa724b2a 100644 --- a/docs/intro/knowledge-base.md +++ b/docs/intro/knowledge-base.md @@ -58,7 +58,6 @@ LightRAG 知识库可在知识库详情中可视化，但不支持在侧边栏 ### 批量脚本 - 上传并入库：参见 `scripts/batch_upload.py upload` -- 转换为 Markdown：参见 `scripts/batch_upload.py trans` ## 知识图谱 diff --git a/scripts/batch_upload.py b/scripts/batch_upload.py index 2a51effd..397b2b0a 100644 --- a/scripts/batch_upload.py +++ b/scripts/batch_upload.py @@ -230,74 +230,6 @@ def save_processed_files(record_file: pathlib.Path, processed_files: set[str]): console.print(f"[bold red]Error: Could not save processed files record: {e}[/bold red]") -async def convert_to_markdown( - client: httpx.AsyncClient, - base_url: str, - db_id: str, - server_file_path: str, -) -> str | None: - """Calls the file-to-markdown conversion endpoint.""" - try: - response = await client.post( - f"{base_url}/knowledge/files/markdown", - json={"db_id": db_id, "file_path": server_file_path}, - timeout=600, # 10 minutes timeout for conversion - ) - response.raise_for_status() - result = response.json() - if result.get("status") == "success": - return result.get("markdown_content") - else: - console.print(f"[bold red]Failed to convert {server_file_path}: {result.get('message')}[/bold red]") - return None - except httpx.HTTPStatusError as e: - console.print( - f"[bold red]Failed to convert {server_file_path}: {e.response.status_code} - {e.response.text}[/bold red]" - ) - return None - except httpx.RequestError as e: - console.print(f"[bold red]Request failed for {server_file_path}: {e}[/bold red]") - return None - - -async def trans_worker( - semaphore: asyncio.Semaphore, - client: httpx.AsyncClient, - base_url: str, - db_id: str, - file_path: pathlib.Path, - output_dir: pathlib.Path, - progress: Progress, - task_id: int, -): - """A worker task that uploads a file and converts it to markdown.""" - async with semaphore: - # 1. Upload file - server_file_path = await upload_file(client, base_url, db_id, file_path) - if not server_file_path: - progress.update(task_id, advance=1, postfix=f"[red]Upload failed for {file_path.name}[/red]") - return file_path, "upload_failed" - - # 2. Convert file to markdown - markdown_content = await convert_to_markdown(client, base_url, db_id, server_file_path) - if not markdown_content: - progress.update(task_id, advance=1, postfix=f"[yellow]Conversion failed for {file_path.name}[/yellow]") - return file_path, "conversion_failed" - - # 3. Save markdown content to output directory - try: - output_path = output_dir / file_path.with_suffix(".md").name - output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, "w", encoding="utf-8") as f: - f.write(markdown_content) - progress.update(task_id, advance=1, postfix=f"[green]Converted {file_path.name}[/green]") - return file_path, "success" - except OSError as e: - console.print(f"[bold red]Error saving markdown for {file_path.name}: {e}[/bold red]") - progress.update(task_id, advance=1, postfix=f"[red]Save failed for {file_path.name}[/red]") - return file_path, "save_failed" - - @app.command() def upload( db_id: str = typer.Option(..., help="The ID of the knowledge base."), @@ -447,91 +379,6 @@ def upload( asyncio.run(run()) -@app.command() -def trans( - db_id: str = typer.Option(..., help="The ID of the knowledge base (for temporary file upload)."), - directory: pathlib.Path = typer.Option( - ..., help="The directory containing files to convert.", exists=True, file_okay=False - ), - output_dir: pathlib.Path = typer.Option("output_markdown", help="The directory to save converted markdown files."), - pattern: str = typer.Option("*.docx", help="The glob pattern for files to convert (e.g., '*.pdf', '*.docx')."), - base_url: str = typer.Option("http://127.0.0.1:5050/api", help="The base URL of the API server."), - username: str = typer.Option(..., help="Admin username for login."), - password: str = typer.Option(..., help="Admin password for login."), - concurrency: int = typer.Option(4, help="The number of concurrent conversion tasks."), - recursive: bool = typer.Option(False, "--recursive", "-r", help="Search for files recursively in subdirectories."), -): - """ - Batch convert files to Markdown format. - """ - console.print(f"[bold green]Starting batch conversion for files in: {directory}[/bold green]") - output_dir.mkdir(parents=True, exist_ok=True) - - # Discover files - glob_method = directory.rglob if recursive else directory.glob - files_to_convert = list(glob_method(pattern)) - if not files_to_convert: - console.print(f"[bold yellow]No files found in '{directory}' matching '{pattern}'. Aborting.[/bold yellow]") - raise typer.Exit() - - # 过滤掉macos的隐藏文件 - files_to_convert = [f for f in files_to_convert if not f.name.startswith("._")] - - console.print(f"Found {len(files_to_convert)} files to convert.") - - async def run(): - async with httpx.AsyncClient() as client: - # Login - token = await login(client, base_url, username, password) - if not token: - raise typer.Exit(code=1) - - client.headers = {"Authorization": f"Bearer {token}"} - - # Setup concurrency and tasks - semaphore = asyncio.Semaphore(concurrency) - tasks = [] - - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - BarColumn(), - TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), - TimeElapsedColumn(), - TextColumn("{task.fields[postfix]}"), - console=console, - transient=True, - ) as progress: - task_id = progress.add_task("[bold blue]Converting...", total=len(files_to_convert), postfix="") - - for file_path in files_to_convert: - task = asyncio.create_task( - trans_worker(semaphore, client, base_url, db_id, file_path, output_dir, progress, task_id) - ) - tasks.append(task) - - results = await asyncio.gather(*tasks) - - # Summarize results - successful_files = [] - failed_files = [] - - for file_path, status in results: - if status == "success": - successful_files.append(file_path) - else: - failed_files.append((file_path, status)) - - console.print("[bold green]Batch conversion complete.[/bold green]") - console.print(f" - [green]Successful:[/green] {len(successful_files)}") - console.print(f" - [red]Failed:[/red] {len(failed_files)}") - if failed_files: - for f, status in failed_files: - console.print(f" - {f} (Reason: {status})") - - asyncio.run(run()) - - """ # Example for upload uv run scripts/batch_upload.py upload \ @@ -544,18 +391,6 @@ uv run scripts/batch_upload.py upload \ --concurrency 4 \ --recursive \ --record-file scripts/tmp/batch_processed_files.txt - -# Example for trans -uv run scripts/batch_upload.py trans \ - --db-id your_kb_id \ - --directory path/to/your/data \ - --output-dir path/to/output_markdown \ - --pattern "*.docx" \ - --base-url http://127.0.0.1:5050/api \ - --username your_username \ - --password your_password \ - --concurrency 4 \ - --recursive """ if __name__ == "__main__": app() diff --git a/src/storage/minio/client.py b/src/storage/minio/client.py index 432efd84..629052da 100644 --- a/src/storage/minio/client.py +++ b/src/storage/minio/client.py @@ -51,8 +51,10 @@ class MinIOClient: host_ip = host_ip.split("://")[-1] host_ip = host_ip.rstrip("/") self.public_endpoint = f"{host_ip}:9000" + logger.debug(f"Docker MinIOClient public_endpoint: {self.public_endpoint}") else: self.public_endpoint = "localhost:9000" + logger.debug(f"Default_client: {self.public_endpoint}") @property def client(self) -> Minio: