diff --git a/.gitignore b/.gitignore index 403ed35b..6aeb1ca2 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,6 @@ logs *.log.* *.db *.lock -!uv.lock tmp cache @@ -29,6 +28,7 @@ cache .idea *.nogit* *.private* +*.local* *.secret* *.pdf diff --git a/docker-compose.yml b/docker-compose.yml index f6cb1b22..f133a893 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,9 @@ services: build: context: . dockerfile: docker/api.Dockerfile + # args: + # http_proxy: http://172.19.13.5:7890 + # https_proxy: http://172.19.13.5:7890 image: yuxi-api:0.1.0 container_name: api-dev working_dir: /app @@ -16,7 +19,7 @@ services: reservations: devices: - driver: nvidia - device_ids: ['1'] + device_ids: ['0'] # 使用GPU 1 capabilities: [gpu] ports: - "5050:5050" @@ -28,8 +31,8 @@ services: - NEO4J_URI=bolt://graph:7687 - NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j} - NEO4J_PASSWORD=${NEO4J_PASSWORD:-0123456789} - - MILVUS_URI=http://milvus:19530 - - MINERU_OCR_URI=http://mineru-api:5051 + - MILVUS_URI=http://127.0.0.1:19530 + - MINERU_OCR_URI=http://127.0.0.1:30000 - MODEL_DIR=/models - RUNNING_IN_DOCKER=true command: uv run uvicorn server.main:app --host 0.0.0.0 --port 5050 --reload @@ -143,61 +146,37 @@ services: networks: - app-network restart: unless-stopped - - mineru-api: + # lastest version: wget https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/compose.yaml + mineru-sglang: build: - context: scripts/mineru-api + context: scripts/mineru-sglang dockerfile: Dockerfile - image: mineru-api:latest - container_name: mineru-api - profiles: - - all + image: mineru-sglang:latest + container_name: mineru-sglang + ports: + - 30000:30000 + environment: + MINERU_MODEL_SOURCE: modelscope + entrypoint: mineru-sglang-server + command: + --host 0.0.0.0 + --port 30000 + ulimits: + memlock: -1 + stack: 67108864 + ipc: host + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:30000/health || exit 1"] deploy: resources: reservations: devices: - driver: nvidia - device_ids: ['0'] + device_ids: ["0"] capabilities: [gpu] - ports: - - "5051:5051" networks: - app-network restart: unless-stopped - command: python -m uvicorn app:app --host 0.0.0.0 --port 5051 --app-dir /app - - - # # lastest version: wget https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/compose.yaml - # mineru-sglang: - # build: - # context: scripts/mineru-sglang - # dockerfile: Dockerfile - # image: mineru-sglang:latest - # container_name: mineru-sglang - # ports: - # - 30000:30000 - # environment: - # MINERU_MODEL_SOURCE: modelscope - # entrypoint: mineru-sglang-server - # command: - # --host 0.0.0.0 - # --port 30000 - # ulimits: - # memlock: -1 - # stack: 67108864 - # ipc: host - # healthcheck: - # test: ["CMD-SHELL", "curl -f http://localhost:30000/health || exit 1"] - # deploy: - # resources: - # reservations: - # devices: - # - driver: nvidia - # device_ids: ["0"] - # capabilities: [gpu] - # networks: - # - app-network - # restart: unless-stopped networks: app-network: diff --git a/docker/api.Dockerfile b/docker/api.Dockerfile index e0c9749b..7c9f967b 100644 --- a/docker/api.Dockerfile +++ b/docker/api.Dockerfile @@ -1,5 +1,5 @@ # 使用基础镜像 -FROM python:3.12 +FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04 COPY --from=ghcr.io/astral-sh/uv:0.7.2 /uv /uvx /bin/ # 设置工作目录 diff --git a/pyproject.toml b/pyproject.toml index a2ababb3..dd108932 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "langchain-deepseek>=0.1.3", "langchain-huggingface>=0.2.0", "langchain-openai>=0.3.14", + "langchain-tavily>=0.2.3", "langchain-together>=0.3.0", "langgraph>=0.3.34", "langgraph-checkpoint-sqlite>=2.0.7", @@ -21,6 +22,7 @@ dependencies = [ "langsmith>=0.3.37", "llama-index>=0.12.33", "llama-index-readers-file>=0.4.7", + "mineru[all]>=2.0.3", "neo4j>=5.28.1", "openai>=1.76.0", "opencv-python-headless>=4.11.0.86", diff --git a/src/agents/tools_factory.py b/src/agents/tools_factory.py index 71d40f95..b93960d5 100644 --- a/src/agents/tools_factory.py +++ b/src/agents/tools_factory.py @@ -3,7 +3,7 @@ import re from collections.abc import Callable from typing import Annotated, Any -from langchain_community.tools.tavily_search import TavilySearchResults +from langchain_tavily import TavilySearch from langchain_core.tools import BaseTool, StructuredTool, tool from pydantic import BaseModel, Field @@ -151,4 +151,4 @@ _TOOLS_REGISTRY = { } if config.enable_web_search: - _TOOLS_REGISTRY["WebSearchWithTavily"] = TavilySearchResults(max_results=10) + _TOOLS_REGISTRY["WebSearchWithTavily"] = TavilySearch(max_results=10) diff --git a/src/core/knowledgebase.py b/src/core/knowledgebase.py index 3e93f6e7..88b7332f 100644 --- a/src/core/knowledgebase.py +++ b/src/core/knowledgebase.py @@ -668,11 +668,11 @@ class KnowledgeBase: logger.info(f"Successfully connected to Milvus at {uri}") return True except MilvusException as e: - logger.error(f"Failed to connect to Milvus: {e}，请检查 milvus 的容器是否正常运行。") - logger.error("如果已退出，请重新启动 `docker restart milvus-standalone-dev`。") + logger.error(f"Failed to connect to Milvus: {e}，请检查 milvus 的容器是否正常运行。{traceback.format_exc()}") + logger.error("如果已退出，请重新启动 `docker restart milvus`。") return False except Exception as e: # Catch other potential errors like requests.exceptions.ConnectionError - logger.error(f"An unexpected error occurred while connecting to Milvus at {uri}: {e}") + logger.error(f"An unexpected error occurred while connecting to Milvus at {uri}: {e}, {traceback.format_exc()}") return False diff --git a/src/plugins/_ocr.py b/src/plugins/_ocr.py index b19aea36..5c36b697 100644 --- a/src/plugins/_ocr.py +++ b/src/plugins/_ocr.py @@ -159,53 +159,26 @@ class OCRPlugin: :param pdf_path: PDF文件路径 :return: 提取的文本 """ - mineru_ocr_uri = os.getenv("MINERU_OCR_URI", "http://localhost:5051") + mineru_ocr_uri = os.getenv("MINERU_OCR_URI", "http://localhost:30000") + mineru_ocr_uri_health = f"{mineru_ocr_uri}/health" import requests import json + from .mineru import parse_doc - health_check_response = requests.get(f"{mineru_ocr_uri}/health", timeout=5) - if health_check_response.status_code != 200 or health_check_response.json().get("status") != "healthy": - logger.error("Mineru OCR service health check failed.") + health_check_response = requests.get(mineru_ocr_uri_health, timeout=5) + if health_check_response.status_code != 200: + logger.error(f"Mineru OCR service health check failed with {mineru_ocr_uri_health}: {health_check_response.json()}") raise RuntimeError("Mineru OCR service health check failed. Please check the log use `docker logs mineru-api`") - # 读取PDF文件 - with open(pdf_path, 'rb') as f: - files = {'file': f} - data = { - 'parse_method': 'ocr', # 使用OCR模式 - 'is_json_md_dump': False, # 不需要保存中间文件 - 'return_layout': False, # 不需要返回布局信息 - 'return_info': False, # 不需要返回额外信息 - 'return_content_list': False, # 不需要返回内容列表 - 'return_images': False, # 不需要返回图片 - } + pdf_path_list = [pdf_path] + output_dir = os.path.join(os.getcwd(), "tmp", "mineru_ocr") - try: - # 发送POST请求到Mineru OCR服务 - response = requests.post( - f"{mineru_ocr_uri}/file_parse", - files=files, - data=data - ) - response.raise_for_status() # 检查响应状态 + pdf_text = parse_doc(pdf_path_list, output_dir, + backend="vlm-sglang-client", + server_url=mineru_ocr_uri)[0] - # 解析响应 - result = response.json() - if 'md_content' in result: - return result['md_content'] - else: - logger.error("Mineru OCR response does not contain md_content") - return "" - - except requests.exceptions.RequestException as e: - logger.error(f"Mineru OCR request failed: {str(e)}") - return "" - except json.JSONDecodeError as e: - logger.error(f"Failed to parse Mineru OCR response: {str(e)}") - return "" - except Exception as e: - logger.error(f"Unexpected error in Mineru OCR processing: {str(e)}") - return "" + logger.debug(f"Mineru OCR result: {pdf_text[:50]}(...) total {len(pdf_text)} characters.") + return pdf_text def get_state(task_id): return GOLBAL_STATE.get(task_id, {}) diff --git a/src/plugins/mineru.py b/src/plugins/mineru.py new file mode 100644 index 00000000..9edc0133 --- /dev/null +++ b/src/plugins/mineru.py @@ -0,0 +1,247 @@ +# Copyright (c) Opendatalab. All rights reserved. +import copy +import json +import os +from pathlib import Path +from tqdm import tqdm + +from mineru.cli.common import convert_pdf_bytes_to_bytes_by_pypdfium2, prepare_env, read_fn +from mineru.data.data_reader_writer import FileBasedDataWriter +from mineru.utils.draw_bbox import draw_layout_bbox, draw_span_bbox +from mineru.utils.enum_class import MakeMode +from mineru.backend.vlm.vlm_analyze import doc_analyze as vlm_doc_analyze +from mineru.backend.pipeline.pipeline_analyze import doc_analyze as pipeline_doc_analyze +from mineru.backend.pipeline.pipeline_middle_json_mkcontent import union_make as pipeline_union_make +from mineru.backend.pipeline.model_json_to_middle_json import result_to_middle_json as pipeline_result_to_middle_json +from mineru.backend.vlm.vlm_middle_json_mkcontent import union_make as vlm_union_make + +from src.utils.logging_config import logger + + +def do_parse( + output_dir, # Output directory for storing parsing results + pdf_file_names: list[str], # List of PDF file names to be parsed + pdf_bytes_list: list[bytes], # List of PDF bytes to be parsed + p_lang_list: list[str], # List of languages for each PDF, default is 'ch' (Chinese) + backend="pipeline", # The backend for parsing PDF, default is 'pipeline' + parse_method="auto", # The method for parsing PDF, default is 'auto' + p_formula_enable=True, # Enable formula parsing + p_table_enable=True, # Enable table parsing + server_url=None, # Server URL for vlm-sglang-client backend + f_draw_layout_bbox=True, # Whether to draw layout bounding boxes + f_draw_span_bbox=True, # Whether to draw span bounding boxes + f_dump_md=True, # Whether to dump markdown files + f_dump_middle_json=True, # Whether to dump middle JSON files + f_dump_model_output=True, # Whether to dump model output files + f_dump_orig_pdf=True, # Whether to dump original PDF files + f_dump_content_list=True, # Whether to dump content list files + f_make_md_mode=MakeMode.MM_MD, # The mode for making markdown content, default is MM_MD + start_page_id=0, # Start page ID for parsing, default is 0 + end_page_id=None, # End page ID for parsing, default is None (parse all pages until the end of the document) +) -> list[str]: + + if backend == "pipeline": + for idx, pdf_bytes in enumerate(pdf_bytes_list): + new_pdf_bytes = convert_pdf_bytes_to_bytes_by_pypdfium2(pdf_bytes, start_page_id, end_page_id) + pdf_bytes_list[idx] = new_pdf_bytes + + infer_results, all_image_lists, all_pdf_docs, lang_list, ocr_enabled_list = pipeline_doc_analyze(pdf_bytes_list, p_lang_list, parse_method=parse_method, formula_enable=p_formula_enable,table_enable=p_table_enable) + + + md_results = [] + for idx, model_list in enumerate(infer_results): + model_json = copy.deepcopy(model_list) + pdf_file_name = pdf_file_names[idx] + local_image_dir, local_md_dir = prepare_env(output_dir, pdf_file_name, parse_method) + image_writer, md_writer = FileBasedDataWriter(local_image_dir), FileBasedDataWriter(local_md_dir) + + images_list = all_image_lists[idx] + pdf_doc = all_pdf_docs[idx] + _lang = lang_list[idx] + _ocr_enable = ocr_enabled_list[idx] + middle_json = pipeline_result_to_middle_json(model_list, images_list, pdf_doc, image_writer, _lang, _ocr_enable, p_formula_enable) + + pdf_info = middle_json["pdf_info"] + + pdf_bytes = pdf_bytes_list[idx] + if f_draw_layout_bbox: + draw_layout_bbox(pdf_info, pdf_bytes, local_md_dir, f"{pdf_file_name}_layout.pdf") + + if f_draw_span_bbox: + draw_span_bbox(pdf_info, pdf_bytes, local_md_dir, f"{pdf_file_name}_span.pdf") + + if f_dump_orig_pdf: + md_writer.write( + f"{pdf_file_name}_origin.pdf", + pdf_bytes, + ) + + if f_dump_md: + image_dir = str(os.path.basename(local_image_dir)) + md_content_str = pipeline_union_make(pdf_info, f_make_md_mode, image_dir) + md_writer.write_string( + f"{pdf_file_name}.md", + md_content_str, + ) + md_results.append(md_content_str) + + if f_dump_content_list: + image_dir = str(os.path.basename(local_image_dir)) + content_list = pipeline_union_make(pdf_info, MakeMode.CONTENT_LIST, image_dir) + md_writer.write_string( + f"{pdf_file_name}_content_list.json", + json.dumps(content_list, ensure_ascii=False, indent=4), + ) + + if f_dump_middle_json: + md_writer.write_string( + f"{pdf_file_name}_middle.json", + json.dumps(middle_json, ensure_ascii=False, indent=4), + ) + + if f_dump_model_output: + md_writer.write_string( + f"{pdf_file_name}_model.json", + json.dumps(model_json, ensure_ascii=False, indent=4), + ) + + logger.info(f"local output dir is {local_md_dir}") + + return md_results + + else: + if backend.startswith("vlm-"): + backend = backend[4:] + + f_draw_span_bbox = False + parse_method = "vlm" + md_results = [] + + for idx, pdf_bytes in enumerate(tqdm(pdf_bytes_list, desc="Parsing documents bytes")): + pdf_file_name = pdf_file_names[idx] + pdf_bytes = convert_pdf_bytes_to_bytes_by_pypdfium2(pdf_bytes, start_page_id, end_page_id) + local_image_dir, local_md_dir = prepare_env(output_dir, pdf_file_name, parse_method) + image_writer, md_writer = FileBasedDataWriter(local_image_dir), FileBasedDataWriter(local_md_dir) + middle_json, infer_result = vlm_doc_analyze(pdf_bytes, image_writer=image_writer, backend=backend, server_url=server_url) + + pdf_info = middle_json["pdf_info"] + + if f_draw_layout_bbox: + draw_layout_bbox(pdf_info, pdf_bytes, local_md_dir, f"{pdf_file_name}_layout.pdf") + + if f_draw_span_bbox: + draw_span_bbox(pdf_info, pdf_bytes, local_md_dir, f"{pdf_file_name}_span.pdf") + + if f_dump_orig_pdf: + md_writer.write( + f"{pdf_file_name}_origin.pdf", + pdf_bytes, + ) + + if f_dump_md: + image_dir = str(os.path.basename(local_image_dir)) + md_content_str = vlm_union_make(pdf_info, f_make_md_mode, image_dir) + md_writer.write_string( + f"{pdf_file_name}.md", + md_content_str, + ) + md_results.append(md_content_str) + + if f_dump_content_list: + image_dir = str(os.path.basename(local_image_dir)) + content_list = vlm_union_make(pdf_info, MakeMode.CONTENT_LIST, image_dir) + md_writer.write_string( + f"{pdf_file_name}_content_list.json", + json.dumps(content_list, ensure_ascii=False, indent=4), + ) + + if f_dump_middle_json: + md_writer.write_string( + f"{pdf_file_name}_middle.json", + json.dumps(middle_json, ensure_ascii=False, indent=4), + ) + + if f_dump_model_output: + model_output = ("\n" + "-" * 50 + "\n").join(infer_result) + md_writer.write_string( + f"{pdf_file_name}_model_output.txt", + model_output, + ) + + logger.info(f"local output dir is {local_md_dir}") + + return md_results + + +def parse_doc( + path_list: list[Path], + output_dir, + lang="ch", + backend="pipeline", + method="auto", + server_url=None, + start_page_id=0, # Start page ID for parsing, default is 0 + end_page_id=None # End page ID for parsing, default is None (parse all pages until the end of the document) +) -> list[str]: + """ + Parameter description: + path_list: List of document paths to be parsed, can be PDF or image files. + output_dir: Output directory for storing parsing results. + lang: Language option, default is 'ch', optional values include['ch', 'ch_server', 'ch_lite', 'en', 'korean', 'japan', 'chinese_cht', 'ta', 'te', 'ka']。 + Input the languages in the pdf (if known) to improve OCR accuracy. Optional. + Adapted only for the case where the backend is set to "pipeline" + backend: the backend for parsing pdf: + pipeline: More general. + vlm-transformers: More general. + vlm-sglang-engine: Faster(engine). + vlm-sglang-client: Faster(client). + without method specified, pipeline will be used by default. + method: the method for parsing pdf: + auto: Automatically determine the method based on the file type. + txt: Use text extraction method. + ocr: Use OCR method for image-based PDFs. + Without method specified, 'auto' will be used by default. + Adapted only for the case where the backend is set to "pipeline". + server_url: When the backend is `sglang-client`, you need to specify the server_url, for example:`http://127.0.0.1:30000` + """ + try: + file_name_list = [] + pdf_bytes_list = [] + lang_list = [] + for path in tqdm(path_list, desc="Parsing documents"): + file_name = str(Path(path).stem) + pdf_bytes = read_fn(path) + file_name_list.append(file_name) + pdf_bytes_list.append(pdf_bytes) + lang_list.append(lang) + + result = do_parse( + output_dir=output_dir, + pdf_file_names=file_name_list, + pdf_bytes_list=pdf_bytes_list, + p_lang_list=lang_list, + backend=backend, + parse_method=method, + server_url=server_url, + start_page_id=start_page_id, + end_page_id=end_page_id + ) + return result if result else [""] + + except Exception as e: + logger.exception(e) + return [""] + +if __name__ == "__main__": + + pdf_files_dir = "/home/zwj/workspace/projects/Yuxi-Know/test/struct_pdf" + output_dir = "/home/zwj/workspace/projects/Yuxi-Know/test/struct_pdf_output" + pdf_suffixes = [".pdf"] + image_suffixes = [".png", ".jpeg", ".jpg"] + + doc_path_list = [] + for doc_path in Path(pdf_files_dir).glob('*'): + if doc_path.suffix in pdf_suffixes + image_suffixes: + doc_path_list.append(doc_path) + + parse_doc(doc_path_list, output_dir, backend="vlm-sglang-client", server_url="http://172.19.13.5:30000") # faster(client). \ No newline at end of file diff --git a/src/utils/logging_config.py b/src/utils/logging_config.py index 26a48417..376564f7 100644 --- a/src/utils/logging_config.py +++ b/src/utils/logging_config.py @@ -3,52 +3,43 @@ import os from datetime import datetime import pytz -from colorlog import ColoredFormatter + +from loguru import logger DATETIME = datetime.now(pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d-%H%M%S') # DATETIME = "debug" # 为了方便，调试的时候输出到 debug.log 文件 LOG_FILE = f'saves/log/project-{DATETIME}.log' - -def setup_logger(name, level=logging.DEBUG, console=True): +def setup_logger(name, level="DEBUG", console=True): + """使用 loguru 设置日志记录器""" os.makedirs("saves/log", exist_ok=True) - - """Function to setup logger with the given name and log file.""" - logger = logging.getLogger(name) - logger.setLevel(level) - - # 清除已有的 Handler，防止重复添加 - if logger.hasHandlers(): - logger.handlers.clear() - - # 文件日志（无颜色） - file_handler = logging.FileHandler(LOG_FILE, encoding='utf-8') - file_handler.setLevel(level) - file_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(name)s - %(message)s') - file_handler.setFormatter(file_formatter) - logger.addHandler(file_handler) - - # 控制台日志（有颜色） + + # 移除默认的 handler + logger.remove() + + # 添加文件日志（无颜色） + logger.add( + LOG_FILE, + level=level, + format="{time:YYYY-MM-DD HH:mm:ss} - {level} - {name} - {message}", + encoding="utf-8", + rotation="10 MB", # 文件大小达到 10MB 时轮转 + retention="30 days", # 保留30天的日志 + compression="zip" # 压缩旧日志文件 + ) + + # 添加控制台日志（有颜色） if console: - console_handler = logging.StreamHandler() - console_handler.setLevel(level) - color_formatter = ColoredFormatter( - "%(log_color)s%(asctime)s - %(levelname)s - %(name)s - %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - log_colors={ - 'DEBUG': 'cyan', - 'INFO': 'green', - 'WARNING': 'yellow', - 'ERROR': 'red', - 'CRITICAL': 'bold_red', - } + logger.add( + lambda msg: print(msg, end=""), + level=level, + format="{time:YYYY-MM-DD HH:mm:ss} - {level} - {name} - {message}", + colorize=True ) - console_handler.setFormatter(color_formatter) - logger.addHandler(console_handler) - + return logger -# Setup the root logger +# 设置根日志记录器 logger = setup_logger('Yuxi') # If you want to disable logging from external libraries diff --git a/web/src/views/DataBaseInfoView.vue b/web/src/views/DataBaseInfoView.vue index 50d8a39e..5a88fad3 100644 --- a/web/src/views/DataBaseInfoView.vue +++ b/web/src/views/DataBaseInfoView.vue @@ -41,6 +41,113 @@ + + + +
+
+

调整分块参数可以控制文本的切分方式，影响检索质量和文档加载效率。

+
+ + + +

每个文本片段的最大字符数

+ + + +

相邻文本片段间的重叠字符数

+ + +
+ + + + + +
+
+
+
+ 上传文件 +
+
+ 输入网址 +
+
+
+ + 分块参数 ({{ chunkParams.chunk_size }}/{{ chunkParams.chunk_overlap }}) + +
+
+ +
+ + + + 启用OCR功能，支持PDF文件的文本提取 + + +
+ + +
+ +

点击或者把文件拖拽到这里上传

+

+ 目前仅支持上传文本文件，如 .pdf, .txt, .md。且同名文件无法重复添加 +

+ +
+ + +
+ + + + + +

+ 支持添加网页内容，系统会自动抓取网页文本并进行分块。请确保URL格式正确且可以公开访问。 +

+
+
+ +
@@ -49,7 +156,7 @@
- 添加文件 + 添加文件 刷新
@@ -73,93 +180,7 @@
-
-
-
-
-

调整分块参数可以控制文本的切分方式，影响检索质量和文档加载效率。

-
- - - -

每个文本片段的最大字符数

- - - -

相邻文本片段间的重叠字符数

- - - -

启用OCR功能，支持PDF文件的文本提取

- - -
-
-
-
-
- 上传文件 -
-
- 输入网址 -
-
- -
- -

点击或者把文件拖拽到这里上传

-

- 目前仅支持上传文本文件，如 .pdf, .txt, .md。且同名文件无法重复添加 -

- -
- - -
- - - - - -

- 支持添加网页内容，系统会自动抓取网页文本并进行分块。请确保URL格式正确且可以公开访问。 -

-
- -
- - 生成分块 - -
-
-
{ }) } -const handleShowAddFilesBlock = () => { - state.showAddFilesBlock = !state.showAddFilesBlock -} + const deleteDatabse = () => { Modal.confirm({ @@ -794,8 +813,7 @@ const chunkFiles = () => { if (data.status === 'success') { message.success(data.message || '文件已提交处理，请稍后在列表刷新查看状态'); fileList.value = []; // 清空已上传文件列表 - // chunkResults.value = []; // 清空旧的预览结果 - // activeFileKeys.value = []; // 清空旧的预览结果 + addFilesModalVisible.value = false; // 关闭弹窗 getDatabaseInfo(); // 刷新数据库信息以显示新文件及其状态 } else { message.error(data.message || '文件处理失败'); @@ -833,8 +851,7 @@ const chunkUrls = () => { if (data.status === 'success') { message.success(data.message || 'URL已提交处理，请稍后在列表刷新查看状态'); urlList.value = ''; // 清空URL输入 - // chunkResults.value = []; // 清空旧的预览结果 - // activeFileKeys.value = []; // 清空旧的预览结果 + addFilesModalVisible.value = false; // 关闭弹窗 getDatabaseInfo(); // 刷新数据库信息以显示新文件及其状态 } else { message.error(data.message || 'URL处理失败'); @@ -928,6 +945,16 @@ const rules = { name: [{ required: true, message: '请输入知识库名称' }] }; +// 分块参数配置弹窗 +const chunkConfigModalVisible = ref(false); +const tempChunkParams = ref({ + chunk_size: 1000, + chunk_overlap: 200, +}); + +// 添加文件弹窗 +const addFilesModalVisible = ref(false); + // 显示编辑对话框 const showEditModal = () => { editForm.name = database.value.name || ''; @@ -964,6 +991,28 @@ const updateDatabaseInfo = async () => { } }; +// 显示分块参数配置弹窗 +const showChunkConfigModal = () => { + tempChunkParams.value = { + chunk_size: chunkParams.value.chunk_size, + chunk_overlap: chunkParams.value.chunk_overlap, + }; + chunkConfigModalVisible.value = true; +}; + +// 处理分块参数配置提交 +const handleChunkConfigSubmit = () => { + chunkParams.value.chunk_size = tempChunkParams.value.chunk_size; + chunkParams.value.chunk_overlap = tempChunkParams.value.chunk_overlap; + chunkConfigModalVisible.value = false; + message.success('分块参数配置已更新'); +}; + +// 显示添加文件弹窗 +const showAddFilesModal = () => { + addFilesModalVisible.value = true; +}; + const handleIndexFile = (fileId) => { if (!fileId) { message.error('无效的文件ID'); @@ -1319,12 +1368,7 @@ const handleBatchIndex = async () => { } } -.upload { - margin-bottom: 20px; - .upload-dragger { - margin: 0px; - } -} + .my-table { button.ant-btn-link { @@ -1400,98 +1444,21 @@ const handleBatchIndex = async () => { } } -.upload-section { - display: flex; - gap: 20px; - - .upload-sidebar { - min-width: 280px; - height: fit-content; - padding: 20px; - background-color: var(--main-light-6); - border-radius: 8px; - border: 1px solid var(--main-light-3); - flex: 0 0 280px; - // box-shadow: 0 2px 8px rgba(0, 0, 0, 0.05); - - .chunking-params { - h4 { - margin-top: 0; - margin-bottom: 16px; - color: var(--main-color); - font-size: 18px; - text-align: center; - font-weight: bold; - padding-bottom: 10px; - border-bottom: 1px dashed var(--main-light-3); - } - - .params-info { - background-color: var(--main-light-4); - border-radius: 6px; - padding: 10px 12px; - margin-bottom: 16px; - - p { - margin: 0; - font-size: 13px; - line-height: 1.5; - color: var(--gray-700); - } - } - - .ant-form-item { - margin-bottom: 16px; - - .ant-form-item-label { - padding-bottom: 6px; - - label { - color: var(--gray-800); - font-weight: 500; - font-size: 15px; - } - } - } - - .ant-input-number { - width: 100%; - border-radius: 6px; - - &:hover, &:focus { - border-color: var(--main-color); - } - } - - .ant-switch { - background-color: var(--gray-400); - - &.ant-switch-checked { - background-color: var(--main-color); - } - } - - // 添加参数说明 - .param-description { - color: var(--gray-600); - font-size: 12px; - margin-top: 4px; - margin-bottom: 0; - } - } - } - - .upload-main { - flex: 1; +.add-files-content { + .upload-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 16px; + gap: 20px; .source-selector { display: flex; gap: 10px; - margin-bottom: 16px; .upload-mode-selector { cursor: pointer; - padding: 8px 16px; + padding: 4px 16px; border-radius: 8px; background-color: var(--main-light-4); border: 1px solid var(--main-light-3); @@ -1503,6 +1470,116 @@ const handleBatchIndex = async () => { } } } + + .config-controls { + .ant-btn { + border-color: var(--main-light-3); + color: var(--gray-700); + &:hover { + border-color: var(--main-color); + color: var(--main-color); + } + } + } + } + + .ocr-config { + margin-bottom: 16px; + padding: 12px 16px; + background-color: var(--main-light-6); + border-radius: 8px; + border: 1px solid var(--main-light-3); + + .ant-form-item { + margin-bottom: 0; + + .ant-form-item-label { + color: var(--gray-800); + font-weight: 500; + } + } + + .param-description { + color: var(--gray-600); + font-size: 12px; + margin-left: 12px; + } + } + + .upload { + margin-bottom: 20px; + .upload-dragger { + margin: 0px; + min-height: 200px; + } + } + + .url-input { + margin-bottom: 20px; + + .ant-textarea { + border-color: var(--main-light-3); + background-color: #fff; + font-family: monospace; + resize: vertical; + } + + .ant-textarea:hover, + .ant-textarea:focus { + border-color: var(--main-color); + } + + .url-hint { + font-size: 13px; + color: var(--gray-600); + margin-top: 5px; + line-height: 1.5; + } + } +} + +.chunk-config-content { + .params-info { + background-color: var(--main-light-4); + border-radius: 6px; + padding: 10px 12px; + margin-bottom: 16px; + + p { + margin: 0; + font-size: 13px; + line-height: 1.5; + color: var(--gray-700); + } + } + + .ant-form-item { + margin-bottom: 16px; + + .ant-form-item-label { + padding-bottom: 6px; + + label { + color: var(--gray-800); + font-weight: 500; + font-size: 15px; + } + } + } + + .ant-input-number { + border-radius: 6px; + + &:hover, &:focus { + border-color: var(--main-color); + } + } + + .param-description { + color: var(--gray-600); + font-size: 12px; + margin-top: 4px; + margin-bottom: 0; } } @@ -1560,24 +1637,7 @@ const handleBatchIndex = async () => { margin-bottom: 20px; } -.url-input .ant-textarea { - border-color: var(--main-light-3); - background-color: #fff; - font-family: monospace; - resize: vertical; -} -.url-input .ant-textarea:hover, -.url-input .ant-textarea:focus { - border-color: var(--main-color); -} - -.url-hint { - font-size: 13px; - color: var(--gray-600); - margin-top: 5px; - line-height: 1.5; -} .loading-container { display: flex;