From 0d39e97763be9220fa96c9f0d23af3157c7b9bf3 Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Mon, 28 Apr 2025 23:48:01 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=8E=AF=E5=A2=83=E9=83=A8?= =?UTF-8?q?=E7=BD=B2=E6=96=B9=E6=B3=95=EF=BC=8C=E9=83=A8=E7=BD=B2=E9=80=9F?= =?UTF-8?q?=E5=BA=A6=E6=9B=B4=E5=BF=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 98 +++++----------------------- docker-compose.yml | 14 ++-- docker/api.Dockerfile | 11 ++-- docker/docker-compose.yml | 131 -------------------------------------- 4 files changed, 29 insertions(+), 225 deletions(-) delete mode 100644 docker/docker-compose.yml diff --git a/README.md b/README.md index 2497f6cf..694cf454 100644 --- a/README.md +++ b/README.md @@ -27,17 +27,13 @@ ## 📋 更新日志 +- **2025.04.27** - 支持 Ollama 等带有 `Content` 标签的消息解析 - **2025.03.30** - 系统中集成智能体（WIP， [PR#96](https://github.com/xerrors/Yuxi-Know/pull/96)） - **2025.02.24** - 新增网页检索以及内容展示，需配置 `TAVILY_API_KEY`，感谢 [littlewwwhite](https://github.com/littlewwwhite) - **2025.02.23** - SiliconFlow 的 Rerank 和 Embedding model 支持，现默认使用 SiliconFlow - **2025.02.20** - DeepSeek-R1 支持，需配置 `DEEPSEEK_API_KEY` 或 `SILICONFLOW_API_KEY` - **2024.10.12** - 后端修改为 [FastAPI](https://github.com/fastapi)，添加 [Milvus-Standalone](https://github.com/milvus-io) 独立部署 -![功能展示](https://github.com/user-attachments/assets/8416a933-cc43-45d0-bf06-00df0ba6c4fb) - -| PC 网页 | 小屏设备 | -| :-------------------------------------------------------------------------------------- | :-------------------------------------------------------------------------------------- | -| ![image](https://github.com/user-attachments/assets/5f3d7e69-baa8-4c59-90fc-391343e59af6) | ![image](https://github.com/user-attachments/assets/51efabce-a097-47fd-9fca-d3b0943af86a) | ### 环境配置 @@ -61,12 +57,11 @@ ZHIPUAI_API_KEY= # 如果配置 智谱清言 添加此行，并替换 **开发环境启动**（源代码修改会自动更新）： ```bash -docker compose -f docker/docker-compose.dev.yml --env-file src/.env up --build +docker compose --env-file src/.env up --build ``` > 添加 `-d` 参数可在后台运行 - 成功启动后，会看到以下容器： ``` @@ -84,51 +79,14 @@ docker compose -f docker/docker-compose.dev.yml --env-file src/.env up --build ### 系统预览 -![系统演示](./images/demo.gif) - -问答支持 Deepseek-R1 等推理模型、知识图谱检索、知识库检索、网页检索 - -![Image](https://github.com/user-attachments/assets/97ad6771-e4a2-4324-a001-ecb0abe40fdd) - -网页检索结果： - -![Image](https://github.com/user-attachments/assets/20d51700-78d5-48a8-ab16-00bc98a1df37) - -知识图谱检索结果： - -![Image](https://github.com/user-attachments/assets/61b4e06f-4e6e-4a75-bfcb-b1a424523bb6) - -知识库检索结果 - -![Image](https://github.com/user-attachments/assets/53e786f4-08d7-45c1-bcec-df46d0c3c49b) - -知识库管理： - - -![Image](https://github.com/user-attachments/assets/55a6de55-59a0-4636-9c00-e4bfd6573c1b) - -![Image](https://github.com/user-attachments/assets/0f4f39ba-e6b7-4f18-a3c3-147477922f58) - - -知识图谱 - -![Image](https://github.com/user-attachments/assets/5849d875-801c-4d29-9d38-9245685e9d73) - -可视化配置： - -![Image](https://github.com/user-attachments/assets/5faac8bc-7968-42d0-a77d-5f8881a24de1) - -丰富的模型支持： - -![Image](https://github.com/user-attachments/assets/7a54ca07-78da-4aef-b0a0-47aa539dae8e) - +> 待补充 ### 服务管理 **关闭服务**： ```bash -docker compose -f docker/docker-compose.dev.yml --env-file src/.env down +docker compose --env-file src/.env down ``` **查看日志**： @@ -143,14 +101,14 @@ docker logs <容器名称> # 例如：docker logs api-dev 本项目支持通过 API 调用的模型，本地模型需使用 vllm、ollama 转成 API 服务后使用。 -| 模型供应商 | 默认模型 | 配置项目 | -| :--------------------- | :---------------------------------- | :--------------------------------------------- | -| `siliconflow` (默认) | `Qwen/Qwen2.5-7B-Instruct` (免费) | `SILICONFLOW_API_KEY` | -| `openai` | `gpt-4o` | `OPENAI_API_KEY` | -| `deepseek` | `deepseek-chat` | `DEEPSEEK_API_KEY` | -| `arc`（豆包方舟） | `doubao-1-5-pro-32k-250115` | `ARK_API_KEY` | -| `zhipu`（智谱清言） | `glm-4-flash` | `ZHIPUAI_API_KEY` | -| `dashscope`（阿里） | `qwen-max-latest` | `DASHSCOPE_API_KEY` | +| 模型供应商 | 默认模型 | 配置项目 | +| :--------------------- | :---------------------------------- | :---------------------- | +| `siliconflow` (默认) | `Qwen/Qwen2.5-7B-Instruct` (免费) | `SILICONFLOW_API_KEY` | +| `openai` | `gpt-4o` | `OPENAI_API_KEY` | +| `deepseek` | `deepseek-chat` | `DEEPSEEK_API_KEY` | +| `arc`（豆包方舟） | `doubao-1-5-pro-32k-250115` | `ARK_API_KEY` | +| `zhipu`（智谱清言） | `glm-4-flash` | `ZHIPUAI_API_KEY` | +| `dashscope`（阿里） | `qwen-max-latest` | `DASHSCOPE_API_KEY` | #### 添加新模型供应商 @@ -189,7 +147,6 @@ ark: ![image](https://github.com/user-attachments/assets/ab62ea17-c7d0-4f94-84af-c4bab26865ad) - **添加向量模型** ```yaml @@ -215,20 +172,9 @@ ark: ## 📚 知识库支持 -本项目支持多种格式的知识库文件： +本项目支持多种格式的知识库文件：PDF、TXT、Markdown、Docx。 -- PDF -- Txt -- Markdown -- Docx - -文件上传后，系统会： - -1. 将文件转换为纯文本 -2. 使用向量模型将文本转换为向量 -3. 存储到向量数据库中 - -> 此过程可能需要一定时间，请耐心等待。 +文件上传后，系统会 对文件进行分块、索引、存储到向量数据库（Milvus）中，此过程可能需要一定时间，请耐心等待。 ## 🕸️ 知识图谱支持 @@ -249,7 +195,7 @@ ark: - 默认账户：`neo4j` - 默认密码：`0123456789` -可在 `docker/docker-compose.yml` 和 `docker/docker-compose.dev.yml` 中修改配置（注意同时修改 `api.environment` 和 `graph.environment`）。 +可在 `docker-compose.yml` 中修改配置（注意同时修改 `api.environment` 和 `graph.environment`）。 目前项目暂不支持同时查询多个知识图谱。如已有基于 neo4j 的知识图谱，可删除 `docker-compose.yml` 中的 `graph` 配置项，并修改 `api.environment` 中的 `NEO4J_URI` 为您的 neo4j 服务地址。同时，需要确保节点的标签中包含 Entity 标签，才能正常触发索引。 @@ -261,20 +207,6 @@ ark: 贡献者名单 -## ❓ 常见问题 - -### 镜像下载问题 - -如无法直接下载相关镜像，可参考 [DaoCloud/public-image-mirror](https://github.com/DaoCloud/public-image-mirror?tab=readme-ov-file#%E5%BF%AB%E9%80%9F%E5%BC%80%E5%A7%8B)，尝试替换前缀： - -```bash -# 以 neo4j 为例，其余类似 -docker pull m.daocloud.io/docker.io/library/neo4j:latest - -# 然后重命名镜像 -docker tag m.daocloud.io/docker.io/library/neo4j:latest neo4j:latest -``` - ## Star History [![Star History Chart](https://api.star-history.com/svg?repos=xerrors/Yuxi-Know)](https://star-history.com/#xerrors/Yuxi-Know) diff --git a/docker-compose.yml b/docker-compose.yml index 9b620cd4..c81e0c2c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,7 +23,7 @@ services: - MILVUS_URI=http://milvus:19530 - MODEL_DIR=/models # 优先级高于 .env 中的 MODEL_DIR - RUNNING_IN_DOCKER=true - command: uvicorn server.main:app --host 0.0.0.0 --port 5050 --reload + command: uv run uvicorn server.main:app --host 0.0.0.0 --port 5050 --reload web: build: @@ -52,8 +52,8 @@ services: - "7474:7474" - "7687:7687" volumes: - - ./volumes/neo4j/data:/data - - ./volumes/neo4j/logs:/var/lib/neo4j/logs + - ./docker/volumes/neo4j/data:/data + - ./docker/volumes/neo4j/logs:/var/lib/neo4j/logs environment: - NEO4J_AUTH=neo4j/0123456789 - NEO4J_server_bolt_listen__address=0.0.0.0:7687 @@ -71,7 +71,7 @@ services: - ETCD_QUOTA_BACKEND_BYTES=4294967296 - ETCD_SNAPSHOT_COUNT=50000 volumes: - - ./volumes/milvus/etcd:/etcd + - ./docker/volumes/milvus/etcd:/etcd command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd healthcheck: test: ["CMD", "etcdctl", "endpoint", "health"] @@ -88,7 +88,7 @@ services: MINIO_ACCESS_KEY: minioadmin MINIO_SECRET_KEY: minioadmin volumes: - - ./volumes/milvus/minio:/minio_data + - ./docker/volumes/milvus/minio:/minio_data command: minio server /minio_data healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] @@ -109,8 +109,8 @@ services: MINIO_ADDRESS: minio:9000 MILVUS_LOG_LEVEL: error # Add this line to reduce log output volumes: - - ./volumes/milvus/milvus:/var/lib/milvus - - ./volumes/milvus/logs:/var/lib/milvus/logs + - ./docker/volumes/milvus/milvus:/var/lib/milvus + - ./docker/volumes/milvus/logs:/var/lib/milvus/logs healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"] interval: 30s diff --git a/docker/api.Dockerfile b/docker/api.Dockerfile index 080582c8..571a5f81 100644 --- a/docker/api.Dockerfile +++ b/docker/api.Dockerfile @@ -4,19 +4,22 @@ FROM python:3.12 # 设置工作目录 WORKDIR /app +# 安装系统依赖 +RUN apt-get update && apt-get install -y \ + python3-dev \ + ffmpeg \ + libsm6 \ + libxext6 + # 复制 requirements.txt 文件（这一步如果文件没变，Docker 会使用缓存） # COPY../requirements.txt /app/requirements.txt COPY ../pyproject.toml /app/pyproject.toml -COPY ../uv.lock /app/uv.lock COPY ../.python-version /app/.python-version # 安装依赖（Docker 会缓存这一步，除非 requirements.txt 发生变化） RUN pip install uv RUN uv sync -RUN apt-get clean -RUN apt-get update && apt-get install ffmpeg libsm6 libxext6 -y - # 复制代码到容器中 COPY ../src /app/src COPY ../server /app/server diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index a9b5a0c5..00000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,131 +0,0 @@ -services: - api: - build: - context: .. - dockerfile: docker/api.Dockerfile - container_name: api-dev - working_dir: /app - volumes: - - ../server:/app/server - - ../src:/app/src - - ../saves_dev:/app/saves - # - ${MODEL_DIR}:/models # 如果出现 undefined volume MODEL_DIR: invalid compose project，请添加此环境变量或者注释此行 - ports: - - "5050:5050" - networks: - - app-network - extra_hosts: - - "host.docker.internal:host-gateway" - environment: - - NEO4J_URI=bolt://graph:7687 - - NEO4J_USERNAME=neo4j - - NEO4J_PASSWORD=0123456789 - - MILVUS_URI=http://milvus:19530 - - MODEL_DIR=/models # 优先级高于 .env 中的 MODEL_DIR - - RUNNING_IN_DOCKER=true - command: uvicorn server.main:app --host 0.0.0.0 --port 5050 --reload - - web: - build: - context: .. - dockerfile: docker/web.Dockerfile - target: development - container_name: web-dev - volumes: - - ../web:/app - - /app/node_modules - ports: - - "5173:5173" - depends_on: - - api - networks: - - app-network - environment: - - NODE_ENV=development - - VITE_API_URL=http://api:5050 # 添加这行 - command: pnpm run server - - graph: - image: neo4j:5.26 - container_name: graph-dev - ports: - - "7474:7474" - - "7687:7687" - volumes: - - ./volumes/neo4j/data:/data - - ./volumes/neo4j/logs:/var/lib/neo4j/logs - environment: - - NEO4J_AUTH=neo4j/0123456789 - - NEO4J_server_bolt_listen__address=0.0.0.0:7687 - - NEO4J_server_http_listen__address=0.0.0.0:7474 - - ENTITY_EMBEDDING=true - networks: - - app-network - - etcd: - container_name: milvus-etcd-dev - image: quay.io/coreos/etcd:v3.5.5 - environment: - - ETCD_AUTO_COMPACTION_MODE=revision - - ETCD_AUTO_COMPACTION_RETENTION=1000 - - ETCD_QUOTA_BACKEND_BYTES=4294967296 - - ETCD_SNAPSHOT_COUNT=50000 - volumes: - - ./volumes/milvus/etcd:/etcd - command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd - healthcheck: - test: ["CMD", "etcdctl", "endpoint", "health"] - interval: 30s - timeout: 20s - retries: 3 - networks: - - app-network - - minio: - container_name: milvus-minio-dev - image: minio/minio:RELEASE.2023-03-20T20-16-18Z - environment: - MINIO_ACCESS_KEY: minioadmin - MINIO_SECRET_KEY: minioadmin - volumes: - - ./volumes/milvus/minio:/minio_data - command: minio server /minio_data - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 30s - timeout: 20s - retries: 3 - networks: - - app-network - - milvus: - image: milvusdb/milvus:v2.5.6 - container_name: milvus-standalone-dev - command: ["milvus", "run", "standalone"] - security_opt: - - seccomp:unconfined - environment: - ETCD_ENDPOINTS: etcd:2379 - MINIO_ADDRESS: minio:9000 - MILVUS_LOG_LEVEL: error # Add this line to reduce log output - volumes: - - ./volumes/milvus/milvus:/var/lib/milvus - - ./volumes/milvus/logs:/var/lib/milvus/logs - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"] - interval: 30s - start_period: 90s - timeout: 20s - retries: 3 - ports: - - "19530:19530" - - "9091:9091" - depends_on: - - "etcd" - - "minio" - networks: - - app-network - -networks: - app-network: - driver: bridge