From 5ed016298c9b4e281f2d7d02960982f16577b380 Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Fri, 18 Jul 2025 11:29:12 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=9B=B4=E6=96=B0=20Docker=20=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=EF=BC=8C=E6=B7=BB=E5=8A=A0=20Milvus=20=E4=BB=A4?= =?UTF-8?q?=E7=89=8C=E6=94=AF=E6=8C=81=EF=BC=8C=E4=BC=98=E5=8C=96=E5=81=A5?= =?UTF-8?q?=E5=BA=B7=E6=A3=80=E6=9F=A5=E5=91=BD=E4=BB=A4=EF=BC=8C=E7=A7=BB?= =?UTF-8?q?=E9=99=A4=20PostgreSQL=20=E9=85=8D=E7=BD=AE=EF=BC=8C=E8=B0=83?= =?UTF-8?q?=E6=95=B4=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F=EF=BC=8C=E5=A2=9E?= =?UTF-8?q?=E5=BC=BA=E7=B3=BB=E7=BB=9F=E7=A8=B3=E5=AE=9A=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yml | 69 +++--- docker/api.Dockerfile | 10 +- docker/init-postgres.sh | 358 ----------------------------- docker/init-postgres.sql | 72 ------ pyproject.toml | 10 - server/routers/graph_router.py | 14 +- src/core/graphbase.py | 12 +- src/core/lightrag_based_kb.py | 30 +-- src/models/__init__.py | 19 ++ src/models/embedding.py | 18 +- web/src/stores/graphStore.js | 16 +- web/src/views/DataBaseInfoView.vue | 25 +- web/src/views/GraphView.vue | 6 +- 13 files changed, 103 insertions(+), 556 deletions(-) delete mode 100644 docker/init-postgres.sh delete mode 100644 docker/init-postgres.sql diff --git a/docker-compose.yml b/docker-compose.yml index c9c7927a..e6c6e7bf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,32 +25,25 @@ services: - NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j} - NEO4J_PASSWORD=${NEO4J_PASSWORD:-0123456789} - MILVUS_URI=${MILVUS_URI:-http://milvus:19530} + - MILVUS_TOKEN=${MILVUS_TOKEN:-} - MINERU_OCR_URI=${MINERU_OCR_URI:-http://mineru:30000} - PADDLEX_URI=${PADDLEX_URI:-http://paddlex-ocr:8080} - MINIO_URI=${MINIO_URI:-http://milvus-minio:9000} - # PostgreSQL 配置 - - POSTGRES_URI=postgresql://${POSTGRES_USER:-lightrag}:${POSTGRES_PASSWORD:-lightrag}@postgres-lightrag:5432/${POSTGRES_DATABASE:-lightrag} - - POSTGRES_HOST=${POSTGRES_HOST:-postgres-lightrag} - - POSTGRES_PORT=${POSTGRES_PORT:-5432} - - POSTGRES_DATABASE=${POSTGRES_DATABASE:-lightrag} - - POSTGRES_USER=${POSTGRES_USER:-lightrag} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-lightrag} - # 连接池配置 - - POSTGRES_MAX_CONNECTIONS=50 - - POSTGRES_MIN_CONNECTIONS=5 - MODEL_DIR=/models - RUNNING_IN_DOCKER=true + - NO_PROXY=localhost,127.0.0.1,milvus,graph,milvus-minio,milvus-etcd-dev,etcd,minio + - no_proxy=localhost,127.0.0.1,milvus,graph,milvus-minio,milvus-etcd-dev,etcd,minio command: uv run uvicorn server.main:app --host 0.0.0.0 --port 5050 --reload restart: unless-stopped healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:5050/health || exit 1"] + test: ["CMD-SHELL", "curl -f http://localhost:5050/api/health || exit 1"] interval: 30s timeout: 15s retries: 8 start_period: 180s depends_on: - postgres: - condition: service_healthy + # postgres: + # condition: service_healthy milvus: condition: service_healthy minio: @@ -80,31 +73,31 @@ services: # PostgreSQL 服务 - 基于官方推荐的 shangor/postgres-for-rag 镜像 # 提供 KV store, VectorDB (pgvector) 和 GraphDB (Apache AGE) 功能 - postgres: - image: shangor/postgres-for-rag:v1.0 - container_name: postgres-lightrag - environment: - - POSTGRES_DATABASE=${POSTGRES_DATABASE:-lightrag} - - POSTGRES_USER=${POSTGRES_USER:-lightrag} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-lightrag} - - POSTGRES_HOST_AUTH_METHOD=md5 - # 启用必要的扩展 - - POSTGRES_INITDB_ARGS=--auth-host=md5 - volumes: - - ./docker/volumes/postgres/data:/var/lib/postgresql/data - - ./docker/init-postgres.sh:/init-postgres.sh - ports: - - "5432:5432" - healthcheck: - test: ["CMD-SHELL", "PGPASSWORD=${POSTGRES_PASSWORD:-lightrag} psql -h localhost -U ${POSTGRES_USER:-lightrag} -d ${POSTGRES_DATABASE:-lightrag} -c \"SELECT ag_catalog.create_graph('test_ready');\" || PGPASSWORD=${POSTGRES_PASSWORD:-lightrag} psql -h localhost -U ${POSTGRES_USER:-lightrag} -d ${POSTGRES_DATABASE:-lightrag} -c 'SELECT 1'"] - interval: 30s - timeout: 15s - retries: 15 - start_period: 120s - networks: - - app-network - restart: unless-stopped - command: ["bash", "/init-postgres.sh"] + # postgres: + # image: shangor/postgres-for-rag:v1.0 + # container_name: postgres-lightrag + # environment: + # - POSTGRES_DATABASE=${POSTGRES_DATABASE:-lightrag} + # - POSTGRES_USER=${POSTGRES_USER:-lightrag} + # - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-lightrag} + # - POSTGRES_HOST_AUTH_METHOD=md5 + # # 启用必要的扩展 + # - POSTGRES_INITDB_ARGS=--auth-host=md5 + # volumes: + # - ./docker/volumes/postgres/data:/var/lib/postgresql/data + # - ./docker/init-postgres.sh:/init-postgres.sh + # ports: + # - "5432:5432" + # healthcheck: + # test: ["CMD-SHELL", "PGPASSWORD=${POSTGRES_PASSWORD:-lightrag} psql -h localhost -U ${POSTGRES_USER:-lightrag} -d ${POSTGRES_DATABASE:-lightrag} -c \"SELECT ag_catalog.create_graph('test_ready');\" || PGPASSWORD=${POSTGRES_PASSWORD:-lightrag} psql -h localhost -U ${POSTGRES_USER:-lightrag} -d ${POSTGRES_DATABASE:-lightrag} -c 'SELECT 1'"] + # interval: 30s + # timeout: 15s + # retries: 15 + # start_period: 120s + # networks: + # - app-network + # restart: unless-stopped + # command: ["bash", "/init-postgres.sh"] graph: image: neo4j:5.26 diff --git a/docker/api.Dockerfile b/docker/api.Dockerfile index e4b1fc4c..ea5779c7 100644 --- a/docker/api.Dockerfile +++ b/docker/api.Dockerfile @@ -29,6 +29,11 @@ COPY ../pyproject.toml /app/pyproject.toml COPY ../.python-version /app/.python-version # 安装依赖项 +ENV HTTP_PROXY=http://172.19.13.5:7890 \ + HTTPS_PROXY=http://172.19.13.5:7890 \ + http_proxy=http://172.19.13.5:7890 \ + https_proxy=http://172.19.13.5:7890 + RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --no-install-project @@ -38,4 +43,7 @@ COPY ../server /app/server # 同步项目 RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync \ No newline at end of file + uv sync + +# 取消代理 +RUN unset HTTP_PROXY HTTPS_PROXY http_proxy https_proxy \ No newline at end of file diff --git a/docker/init-postgres.sh b/docker/init-postgres.sh deleted file mode 100644 index f4227a03..00000000 --- a/docker/init-postgres.sh +++ /dev/null @@ -1,358 +0,0 @@ -#!/bin/bash - -# PostgreSQL 初始化脚本 -# 适用于 shangor/postgres-for-rag 镜像 - -set -e - -echo "Starting PostgreSQL initialization..." - -# 启动 PostgreSQL 服务 -service postgresql start -sleep 10 - -echo "PostgreSQL service started, waiting for it to be ready..." - -# 等待 PostgreSQL 完全启动 -while ! su - postgres -c "pg_isready" > /dev/null 2>&1; do - echo "Waiting for PostgreSQL to be ready..." - sleep 5 -done - -# 额外等待以确保数据库完全启动 -echo "PostgreSQL is ready, waiting additional time for full initialization..." -sleep 10 - -echo "PostgreSQL is ready, creating user and database..." - -# 创建用户（如果不存在） -if ! su - postgres -c "psql -tAc \"SELECT 1 FROM pg_roles WHERE rolname='${POSTGRES_USER:-lightrag}'\"" | grep -q 1; then - echo "Creating user ${POSTGRES_USER:-lightrag}..." - su - postgres -c "createuser -s ${POSTGRES_USER:-lightrag}" -else - echo "User ${POSTGRES_USER:-lightrag} already exists" -fi - -# 创建数据库（如果不存在） -if ! su - postgres -c "psql -tAc \"SELECT 1 FROM pg_database WHERE datname='${POSTGRES_DATABASE:-lightrag}'\"" | grep -q 1; then - echo "Creating database ${POSTGRES_DATABASE:-lightrag}..." - su - postgres -c "createdb -O ${POSTGRES_USER:-lightrag} ${POSTGRES_DATABASE:-lightrag}" -else - echo "Database ${POSTGRES_DATABASE:-lightrag} already exists" -fi - -# 设置密码 -echo "Setting password for user ${POSTGRES_USER:-lightrag}..." -su - postgres -c "psql -c \"ALTER USER ${POSTGRES_USER:-lightrag} PASSWORD '${POSTGRES_PASSWORD:-lightrag}';\"" - -# 配置 pg_hba.conf 以允许密码认证 -echo "Configuring PostgreSQL authentication..." -PG_HBA_CONF="/etc/postgresql/16/main/pg_hba.conf" -# 备份原始配置 -cp $PG_HBA_CONF $PG_HBA_CONF.backup - -# 创建新的 pg_hba.conf 文件，确保 lightrag 用户的规则在前面 -cat > $PG_HBA_CONF << EOF -# PostgreSQL Client Authentication Configuration File -# =================================================== - -# TYPE DATABASE USER ADDRESS METHOD - -# LightRAG user authentication (must be first to take precedence) -local ${POSTGRES_DATABASE:-lightrag} ${POSTGRES_USER:-lightrag} md5 -host ${POSTGRES_DATABASE:-lightrag} ${POSTGRES_USER:-lightrag} 127.0.0.1/32 md5 -host ${POSTGRES_DATABASE:-lightrag} ${POSTGRES_USER:-lightrag} ::1/128 md5 -host ${POSTGRES_DATABASE:-lightrag} ${POSTGRES_USER:-lightrag} 0.0.0.0/0 md5 - -# Default configurations -local all postgres peer -local all all peer -host all all 127.0.0.1/32 scram-sha-256 -host all all ::1/128 scram-sha-256 -local replication all peer -host replication all 127.0.0.1/32 scram-sha-256 -host replication all ::1/128 scram-sha-256 -EOF - -# 重启 PostgreSQL 以应用 pg_hba.conf 更改 -echo "Restarting PostgreSQL to apply authentication changes..." -service postgresql restart -sleep 5 - -echo "Installing extensions..." - -# 安装 vector 扩展 -echo "Installing vector extension..." -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \"CREATE EXTENSION IF NOT EXISTS vector;\"" - -# 安装 age 扩展 -echo "Installing age extension..." -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \"CREATE EXTENSION IF NOT EXISTS age;\"" - -echo "Configuring AGE extension..." - -# 加载 AGE 扩展 -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \"LOAD 'age';\"" - -# 设置搜索路径 -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \"SET search_path = ag_catalog, public;\"" - -# 创建图（如果不存在） -echo "Creating LightRAG graph..." -if su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -tAc \"SELECT 1 FROM ag_catalog.ag_graph WHERE name='lightrag'\"" | grep -q 1; then - echo "Graph 'lightrag' already exists" -else - su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \"SELECT ag_catalog.create_graph('lightrag');\"" - echo "Graph 'lightrag' created successfully" -fi - -echo "Creating LightRAG application tables..." - -# 创建 LightRAG 需要的基础表 -echo "Creating lightrag_kv_store table..." -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \" -CREATE TABLE IF NOT EXISTS lightrag_kv_store ( - key VARCHAR PRIMARY KEY, - value JSONB, - created_at TIMESTAMP DEFAULT NOW(), - updated_at TIMESTAMP DEFAULT NOW() -); -\"" - -echo "Creating lightrag_doc_status table..." -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \" -CREATE TABLE IF NOT EXISTS lightrag_doc_status ( - doc_id VARCHAR PRIMARY KEY, - status VARCHAR NOT NULL, - metadata JSONB, - created_at TIMESTAMP DEFAULT NOW(), - updated_at TIMESTAMP DEFAULT NOW() -); -\"" - -echo "Creating indexes for performance optimization..." - -# 为应用表创建索引 -echo "Creating KV store indexes..." -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \" -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_lightrag_kv_store_updated_at ON lightrag_kv_store(updated_at); -\"" - -echo "Creating doc status indexes..." -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \" -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_lightrag_doc_status_status ON lightrag_doc_status(status); -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_lightrag_doc_status_updated_at ON lightrag_doc_status(updated_at); -\"" - -# 等待图表创建完成后创建 AGE 相关索引 -echo "Waiting for graph tables to be fully initialized..." -sleep 15 - -# 验证 AGE 扩展是否正确加载 -echo "Verifying AGE extension is working..." -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \"SELECT ag_catalog.create_graph('test_graph'); SELECT ag_catalog.drop_graph('test_graph', true);\"" || echo "AGE extension verification failed, but continuing..." - -echo "Creating AGE performance indexes..." - -# 检查并创建 Entity 表索引 -echo "Creating Entity table indexes..." -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \" -DO \$\$ -BEGIN - -- Entity 表索引 - IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'lightrag' AND table_name = 'Entity') THEN - PERFORM 1; - -- 主键索引通常自动创建，这里创建其他有用的索引 - IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'lightrag' AND tablename = 'Entity' AND indexname = 'entity_node_id_gin_idx') THEN - CREATE INDEX CONCURRENTLY entity_node_id_gin_idx ON lightrag.\\\"Entity\\\" USING gin(properties); - END IF; - END IF; -EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Entity table not ready yet, skipping indexes'; -END -\$\$; -\"" - -# 检查并创建 DIRECTED 关系表索引 -echo "Creating DIRECTED relationship indexes..." -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \" -DO \$\$ -BEGIN - -- DIRECTED 关系表索引 - IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'lightrag' AND table_name = 'DIRECTED') THEN - IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'lightrag' AND tablename = 'DIRECTED' AND indexname = 'directed_eid_idx') THEN - CREATE INDEX CONCURRENTLY directed_eid_idx ON lightrag.\\\"DIRECTED\\\" (end_id); - END IF; - IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'lightrag' AND tablename = 'DIRECTED' AND indexname = 'directed_sid_idx') THEN - CREATE INDEX CONCURRENTLY directed_sid_idx ON lightrag.\\\"DIRECTED\\\" (start_id); - END IF; - IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'lightrag' AND tablename = 'DIRECTED' AND indexname = 'directed_seid_idx') THEN - CREATE INDEX CONCURRENTLY directed_seid_idx ON lightrag.\\\"DIRECTED\\\" (start_id, end_id); - END IF; - END IF; -EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'DIRECTED table not ready yet, skipping indexes'; -END -\$\$; -\"" - -# 检查并创建 Edge 表索引 -echo "Creating Edge table indexes..." -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \" -DO \$\$ -BEGIN - -- Edge 表索引 - IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'lightrag' AND table_name = '_ag_label_edge') THEN - IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'lightrag' AND tablename = '_ag_label_edge' AND indexname = 'edge_sid_idx') THEN - CREATE INDEX CONCURRENTLY edge_sid_idx ON lightrag.\\\"_ag_label_edge\\\" (start_id); - END IF; - IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'lightrag' AND tablename = '_ag_label_edge' AND indexname = 'edge_eid_idx') THEN - CREATE INDEX CONCURRENTLY edge_eid_idx ON lightrag.\\\"_ag_label_edge\\\" (end_id); - END IF; - IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'lightrag' AND tablename = '_ag_label_edge' AND indexname = 'edge_seid_idx') THEN - CREATE INDEX CONCURRENTLY edge_seid_idx ON lightrag.\\\"_ag_label_edge\\\" (start_id, end_id); - END IF; - END IF; -EXCEPTION - WHEN OTHERS THEN - RAISE NOTICE 'Edge table not ready yet, skipping indexes'; -END -\$\$; -\"" - -echo "Setting up permissions..." - -# 授权给 lightrag 用户 -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \" -GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO ${POSTGRES_USER:-lightrag}; -GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO ${POSTGRES_USER:-lightrag}; -GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA public TO ${POSTGRES_USER:-lightrag}; -GRANT USAGE ON SCHEMA public TO ${POSTGRES_USER:-lightrag}; -\"" - -# 为 AGE 图授权 -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \" -GRANT ALL PRIVILEGES ON SCHEMA lightrag TO ${POSTGRES_USER:-lightrag}; -GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA lightrag TO ${POSTGRES_USER:-lightrag}; -GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA lightrag TO ${POSTGRES_USER:-lightrag}; -GRANT USAGE ON SCHEMA lightrag TO ${POSTGRES_USER:-lightrag}; -\"" - -echo "Applying PostgreSQL performance configuration..." - -# 应用性能配置 -su - postgres -c "psql -c \"ALTER SYSTEM SET shared_preload_libraries = 'age';\"" -su - postgres -c "psql -c \"ALTER SYSTEM SET max_connections = 200;\"" -su - postgres -c "psql -c \"ALTER SYSTEM SET shared_buffers = '256MB';\"" -su - postgres -c "psql -c \"ALTER SYSTEM SET effective_cache_size = '1GB';\"" -su - postgres -c "psql -c \"ALTER SYSTEM SET maintenance_work_mem = '64MB';\"" -su - postgres -c "psql -c \"ALTER SYSTEM SET random_page_cost = 1.1;\"" -su - postgres -c "psql -c \"ALTER SYSTEM SET effective_io_concurrency = 200;\"" -su - postgres -c "psql -c \"ALTER SYSTEM SET work_mem = '8MB';\"" - -# 连接和查询超时配置 -su - postgres -c "psql -c \"ALTER SYSTEM SET statement_timeout = '300000';\"" # 5分钟查询超时 -su - postgres -c "psql -c \"ALTER SYSTEM SET idle_in_transaction_session_timeout = '600000';\"" # 10分钟事务超时 -su - postgres -c "psql -c \"ALTER SYSTEM SET tcp_keepalives_idle = '600';\"" # TCP keepalive 设置 -su - postgres -c "psql -c \"ALTER SYSTEM SET tcp_keepalives_interval = '30';\"" -su - postgres -c "psql -c \"ALTER SYSTEM SET tcp_keepalives_count = '3';\"" - -# 查询优化配置 -su - postgres -c "psql -c \"ALTER SYSTEM SET max_parallel_workers_per_gather = '2';\"" -su - postgres -c "psql -c \"ALTER SYSTEM SET max_parallel_workers = '4';\"" -su - postgres -c "psql -c \"ALTER SYSTEM SET enable_parallel_hash = 'on';\"" - -# 内存和查询限制优化 -su - postgres -c "psql -c \"ALTER SYSTEM SET temp_buffers = '16MB';\"" -su - postgres -c "psql -c \"ALTER SYSTEM SET max_stack_depth = '2MB';\"" -su - postgres -c "psql -c \"ALTER SYSTEM SET huge_pages = 'try';\"" - -# 启用查询计划缓存和日志记录 -su - postgres -c "psql -c \"ALTER SYSTEM SET plan_cache_mode = 'auto';\"" -su - postgres -c "psql -c \"ALTER SYSTEM SET log_statement = 'all';\"" -su - postgres -c "psql -c \"ALTER SYSTEM SET log_duration = 'on';\"" -su - postgres -c "psql -c \"ALTER SYSTEM SET log_min_duration_statement = '1000';\"" # 记录超过1秒的查询 -su - postgres -c "psql -c \"ALTER SYSTEM SET log_disconnections = 'on';\"" -su - postgres -c "psql -c \"ALTER SYSTEM SET log_connections = 'on';\"" - -# 重新加载配置 -su - postgres -c "psql -c \"SELECT pg_reload_conf();\"" - -echo "Restarting PostgreSQL to apply configuration changes..." -service postgresql restart -sleep 5 - -echo "Verifying setup..." - -# 验证扩展 -echo "Installed extensions:" -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \"\\dx\"" - -# 验证图 -echo "Available graphs:" -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \"SELECT name FROM ag_catalog.ag_graph;\"" - -# 验证表 -echo "Created tables:" -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \" -SELECT schemaname, tablename -FROM pg_tables -WHERE schemaname IN ('public', 'lightrag') -ORDER BY schemaname, tablename; -\"" - -# 验证索引 -echo "Created indexes:" -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \" -SELECT schemaname, tablename, indexname, indexdef -FROM pg_indexes -WHERE schemaname IN ('public', 'lightrag') -ORDER BY schemaname, tablename, indexname; -\"" - -# 测试连接 -echo "Testing database connection..." -export PGPASSWORD="${POSTGRES_PASSWORD:-lightrag}" -if PGPASSWORD="${POSTGRES_PASSWORD:-lightrag}" psql -h localhost -p 5432 -U ${POSTGRES_USER:-lightrag} -d ${POSTGRES_DATABASE:-lightrag} -c "SELECT version();" > /dev/null 2>&1; then - echo "✅ Database connection test successful!" -else - echo "❌ Database connection test failed!" - echo "Checking pg_hba.conf..." - cat $PG_HBA_CONF | head -20 - echo "Checking PostgreSQL logs..." - tail -20 /var/log/postgresql/postgresql-16-main.log -fi - -echo "✅ PostgreSQL setup completed successfully!" -echo "Database: ${POSTGRES_DATABASE:-lightrag}" -echo "User: ${POSTGRES_USER:-lightrag}" -echo "Extensions: vector, age" -echo "Graph: lightrag" -echo "Tables: lightrag_kv_store, lightrag_doc_status + AGE graph tables" -echo "Connection: postgresql://${POSTGRES_USER:-lightrag}:***@localhost:5432/${POSTGRES_DATABASE:-lightrag}" - -# 最终验证数据库完全准备就绪 -echo "Performing final readiness check..." -sleep 5 -if PGPASSWORD="${POSTGRES_PASSWORD:-lightrag}" psql -h localhost -p 5432 -U ${POSTGRES_USER:-lightrag} -d ${POSTGRES_DATABASE:-lightrag} -c "SELECT ag_catalog.create_graph('final_test'); SELECT ag_catalog.drop_graph('final_test', true);" > /dev/null 2>&1; then - echo "✅ Database is fully ready for LightRAG operations!" -else - echo "⚠️ Database basic operations work, but AGE may need more time to be fully ready" -fi - -# 显示数据库统计信息 -echo "Database statistics:" -su - postgres -c "psql -d ${POSTGRES_DATABASE:-lightrag} -c \" -SELECT - schemaname, - COUNT(*) as table_count -FROM pg_tables -WHERE schemaname IN ('public', 'lightrag') -GROUP BY schemaname; -\"" - -# 保持容器运行并显示日志 -echo "PostgreSQL is ready for connections on port 5432" -tail -f /var/log/postgresql/postgresql-16-main.log \ No newline at end of file diff --git a/docker/init-postgres.sql b/docker/init-postgres.sql deleted file mode 100644 index b9f2b95a..00000000 --- a/docker/init-postgres.sql +++ /dev/null @@ -1,72 +0,0 @@ --- LightRAG PostgreSQL 初始化脚本 --- 基于官方文档建议: https://github.com/HKUDS/LightRAG - --- 创建必要的扩展 -CREATE EXTENSION IF NOT EXISTS vector; -CREATE EXTENSION IF NOT EXISTS age; - --- 加载 AGE 扩展 -LOAD 'age'; - --- 设置搜索路径 -SET search_path = ag_catalog, "$user", public; - --- 创建 LightRAG 图（如果不存在） --- 注意：这里使用 'lightrag' 作为图名，可以根据需要修改 -SELECT create_graph('lightrag'); - --- 为 AGE 创建索引以提高性能 --- 替换下面的 'lightrag' 为您的实际图名 - --- Entity 表索引 -CREATE INDEX CONCURRENTLY IF NOT EXISTS entity_p_idx ON lightrag."Entity" (id); -CREATE INDEX CONCURRENTLY IF NOT EXISTS entity_idx_node_id ON lightrag."Entity" (ag_catalog.agtype_access_operator(properties, '"node_id"'::agtype)); -CREATE INDEX CONCURRENTLY IF NOT EXISTS entity_node_id_gin_idx ON lightrag."Entity" using gin(properties); - --- Vertex 表索引 -CREATE INDEX CONCURRENTLY IF NOT EXISTS vertex_p_idx ON lightrag."_ag_label_vertex" (id); -CREATE INDEX CONCURRENTLY IF NOT EXISTS vertex_idx_node_id ON lightrag."_ag_label_vertex" (ag_catalog.agtype_access_operator(properties, '"node_id"'::agtype)); - --- DIRECTED 关系表索引 -CREATE INDEX CONCURRENTLY IF NOT EXISTS directed_p_idx ON lightrag."DIRECTED" (id); -CREATE INDEX CONCURRENTLY IF NOT EXISTS directed_eid_idx ON lightrag."DIRECTED" (end_id); -CREATE INDEX CONCURRENTLY IF NOT EXISTS directed_sid_idx ON lightrag."DIRECTED" (start_id); -CREATE INDEX CONCURRENTLY IF NOT EXISTS directed_seid_idx ON lightrag."DIRECTED" (start_id,end_id); - --- Edge 表索引 -CREATE INDEX CONCURRENTLY IF NOT EXISTS edge_p_idx ON lightrag."_ag_label_edge" (id); -CREATE INDEX CONCURRENTLY IF NOT EXISTS edge_sid_idx ON lightrag."_ag_label_edge" (start_id); -CREATE INDEX CONCURRENTLY IF NOT EXISTS edge_eid_idx ON lightrag."_ag_label_edge" (end_id); -CREATE INDEX CONCURRENTLY IF NOT EXISTS edge_seid_idx ON lightrag."_ag_label_edge" (start_id,end_id); - --- 集群表以提高性能 -ALTER TABLE lightrag."DIRECTED" CLUSTER ON directed_sid_idx; - --- 创建 LightRAG 需要的基础表（如果使用 PostgreSQL 作为 KV store） -CREATE TABLE IF NOT EXISTS lightrag_kv_store ( - key VARCHAR PRIMARY KEY, - value JSONB, - created_at TIMESTAMP DEFAULT NOW(), - updated_at TIMESTAMP DEFAULT NOW() -); - --- 创建索引 -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_lightrag_kv_store_updated_at ON lightrag_kv_store(updated_at); - --- 创建文档状态表 -CREATE TABLE IF NOT EXISTS lightrag_doc_status ( - doc_id VARCHAR PRIMARY KEY, - status VARCHAR NOT NULL, - metadata JSONB, - created_at TIMESTAMP DEFAULT NOW(), - updated_at TIMESTAMP DEFAULT NOW() -); - --- 授权给 lightrag 用户 -GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO lightrag; -GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO lightrag; -GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA public TO lightrag; - --- 为 AGE 图授权 -GRANT ALL PRIVILEGES ON SCHEMA lightrag TO lightrag; -GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA lightrag TO lightrag; \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 3830fe67..65ff10bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,6 @@ dependencies = [ "dashscope>=1.23.2", "docx2txt>=0.9", "fastapi>=0.115.12", - "graspologic>=3.3.0", "langchain-community>=0.3.22", "langchain-deepseek>=0.1.3", "langchain-huggingface>=0.2.0", @@ -42,7 +41,6 @@ dependencies = [ "tavily-python>=0.7.0", "unstructured>=0.17.2", "uvicorn[standard]>=0.34.2", - "zhipuai>=2.1.5.20250421", ] [tool.ruff] line-length = 210 # 代码最大行宽 @@ -59,11 +57,3 @@ dev = [ "ruff>=0.12.1", "vllm>=0.8.5.post1", ] - -# uv 配置 - 使用清华大学镜像源 -[tool.uv] -index-url = "https://pypi.tuna.tsinghua.edu.cn/simple/" -extra-index-url = [ - "https://mirrors.aliyun.com/pypi/simple/", - "https://pypi.douban.com/simple/" -] diff --git a/server/routers/graph_router.py b/server/routers/graph_router.py index 00eb546d..2068b1d4 100644 --- a/server/routers/graph_router.py +++ b/server/routers/graph_router.py @@ -1,8 +1,4 @@ -import json -import os -import asyncio import traceback -from typing import List, Dict, Any, Optional from fastapi import APIRouter, Query, HTTPException, Depends from server.utils.auth_middleware import get_admin_user from server.models.user_model import User @@ -23,13 +19,13 @@ async def get_subgraph( ): """ 使用 LightRAG 原生方法获取知识图谱子图 - + Args: db_id: LightRAG 数据库实例ID node_label: 节点标签，用于查找起始节点，使用 "*" 获取全图 max_depth: 子图的最大深度 max_nodes: 返回的最大节点数量 - + Returns: 包含节点和边的知识图谱数据 """ @@ -95,10 +91,10 @@ async def get_graph_labels( ): """ 获取知识图谱中的所有标签 - + Args: db_id: LightRAG 数据库实例ID - + Returns: 图谱中所有可用的标签列表 """ @@ -132,7 +128,7 @@ async def get_available_databases( ): """ 获取所有可用的 LightRAG 数据库 - + Returns: 可用的数据库列表 """ diff --git a/src/core/graphbase.py b/src/core/graphbase.py index 3a5fcd5a..67c7f23f 100644 --- a/src/core/graphbase.py +++ b/src/core/graphbase.py @@ -7,7 +7,7 @@ from neo4j import GraphDatabase as GD from neo4j import Query from src import config -from src.models.embedding import get_embedding_model +from src.models import select_embedding_model from src.utils import logger warnings.filterwarnings("ignore", category=UserWarning) @@ -22,7 +22,7 @@ class GraphDatabase: self.status = "closed" self.kgdb_name = "neo4j" self.embed_model_name = os.getenv("GRAPH_EMBED_MODEL_NAME") or "siliconflow/BAAI/bge-m3" - self.embed_model = get_embedding_model(self.embed_model_name) + self.embed_model = select_embedding_model(self.embed_model_name) self.work_dir = os.path.join(config.save_dir, "knowledge_graph", self.kgdb_name) os.makedirs(self.work_dir, exist_ok=True) @@ -590,9 +590,11 @@ class GraphDatabase: source_id = source.element_id target_id = target.element_id - assert node_dict is not None, "node_dict is required" - source_name = node_dict[source_id]["name"] if source_name is None else source_name - target_name = node_dict[target_id]["name"] if target_name is None else target_name + # 如果没有提供 source_name 或 target_name，则需要 node_dict + if source_name is None or target_name is None: + assert node_dict is not None, "node_dict is required when source_name or target_name is None" + source_name = node_dict[source_id]["name"] if source_name is None else source_name + target_name = node_dict[target_id]["name"] if target_name is None else target_name relationship_type = relationship._properties.get("type", "unknown") if relationship_type == "unknown": diff --git a/src/core/lightrag_based_kb.py b/src/core/lightrag_based_kb.py index 68b7c109..2292f049 100644 --- a/src/core/lightrag_based_kb.py +++ b/src/core/lightrag_based_kb.py @@ -92,11 +92,12 @@ class LightRagBasedKB: # 使用配置的 LLM 和 embedding 函数 rag = LightRAG( working_dir=working_dir, + workspace=db_id, llm_model_func=self._get_llm_func(llm_info), embedding_func=self._get_embedding_func(embed_info), vector_storage="MilvusVectorDBStorage", kv_storage="JsonKVStorage", - graph_storage="PGGraphStorage", + graph_storage="Neo4JStorage", doc_status_storage="JsonDocStatusStorage", log_file_path=os.path.join(self.work_dir, db_id, "lightrag.log"), ) @@ -127,16 +128,16 @@ class LightRagBasedKB: # provider_info = config.model_names[llm_info.get("provider")] # api_key = os.getenv(provider_info.get("env")[0] or "OPENAI_API_KEY") or "no_api_key" # base_url = get_docker_safe_url(provider_info.get("base_url", "http://localhost:8081/v1")) - from src.models import get_custom_model - llm_info = get_custom_model("qwen3:32b-RFnC") + from src.models import select_model + model = select_model("dashscope", "qwen-max-latest") async def llm_model_func(prompt, system_prompt=None, history_messages=[], **kwargs): return await openai_complete_if_cache( - llm_info.get("name", "qwen3-1.7b"), - prompt, + model=model.model_name, + prompt=prompt, system_prompt=system_prompt, history_messages=history_messages, - api_key=llm_info.get("api_key"), - base_url=get_docker_safe_url(llm_info.get("api_base")), + api_key=model.api_key, + base_url=model.base_url, extra_body={"enable_thinking": False}, **kwargs, ) @@ -144,16 +145,16 @@ class LightRagBasedKB: def _get_embedding_func(self, embed_info: dict): """获取 embedding 函数""" - api_key = os.getenv(embed_info.get("api_key", "OPENAI_API_KEY")) or "no_api_key" - base_url = embed_info.get("base_url", "http://localhost:8081/v1").replace("/embeddings", "") + from src.models import select_embedding_model + model = select_embedding_model("siliconflow/BAAI/bge-m3") return EmbeddingFunc( - embedding_dim=embed_info.get("dimension") or 1024, + embedding_dim=model.dimension, max_token_size=4096, func=lambda texts: openai_embed( texts=texts, - model=embed_info.get("model_name") or "Qwen3-Embedding-0.6B", - api_key=api_key, - base_url=get_docker_safe_url(base_url) + model=model.model, + api_key=model.api_key, + base_url=model.base_url.replace("/embeddings", ""), ), ) @@ -335,7 +336,8 @@ class LightRagBasedKB: # 根据内容类型处理内容 if content_type == "file": markdown_content = await self._process_file_to_markdown(item, params=params) - logger.info(f"Markdown content: {markdown_content[:100].replace('\n', ' ')}...") + markdown_content_lines = markdown_content[:100].replace('\n', ' ') + logger.info(f"Markdown content: {markdown_content_lines}...") else: # URL markdown_content = await self._process_url_to_markdown(item, params=params) diff --git a/src/models/__init__.py b/src/models/__init__.py index 885ebdb1..87316f2a 100644 --- a/src/models/__init__.py +++ b/src/models/__init__.py @@ -4,6 +4,7 @@ import traceback from src import config from src.utils.logging_config import logger from src.models.chat_model import OpenAIBase +from src.models.embedding import OllamaEmbedding, OtherEmbedding def select_model(model_provider, model_name=None): """根据模型提供者选择模型""" @@ -38,6 +39,24 @@ def select_model(model_provider, model_name=None): except Exception as e: raise ValueError(f"Model provider {model_provider} load failed, {e} \n {traceback.format_exc()}") + +def select_embedding_model(model_id): + provider, model_name = model_id.split('/', 1) if model_id else ("", "") + support_embed_models = config.embed_model_names.keys() + assert model_id in support_embed_models, f"Unsupported embed model: {model_id}, only support {support_embed_models}" + logger.debug(f"Loading embedding model {model_id}") + if provider == "local": + raise ValueError("Local embedding model is not supported, please use other embedding models") + + elif provider == "ollama": + model = OllamaEmbedding(model_id) + + else: + model = OtherEmbedding(model_id) + + return model + + def get_custom_model(model_id): """return model_info""" assert config.custom_models is not None, "custom_models is not set" diff --git a/src/models/embedding.py b/src/models/embedding.py index 9cfb9afb..02bac5a5 100644 --- a/src/models/embedding.py +++ b/src/models/embedding.py @@ -3,7 +3,6 @@ import json import requests import asyncio from abc import abstractmethod -from zhipuai import ZhipuAI from langchain_huggingface import HuggingFaceEmbeddings from src import config @@ -19,6 +18,7 @@ class BaseEmbeddingModel: self.model = self.info["name"] self.dimension = self.info.get("dimension", None) self.url = get_docker_safe_url(self.info["base_url"]) + self.base_url = get_docker_safe_url(self.info["base_url"]) self.api_key = os.getenv(self.info["api_key"], self.info["api_key"]) @abstractmethod @@ -110,19 +110,3 @@ class OtherEmbedding(BaseEmbeddingModel): "model": self.model, "input": message, } - -def get_embedding_model(model_id): - provider, model_name = model_id.split('/', 1) if model_id else ("", "") - support_embed_models = config.embed_model_names.keys() - assert model_id in support_embed_models, f"Unsupported embed model: {model_id}, only support {support_embed_models}" - logger.debug(f"Loading embedding model {model_id}") - if provider == "local": - raise ValueError("Local embedding model is not supported, please use other embedding models") - - elif provider == "ollama": - model = OllamaEmbedding(model_id) - - else: - model = OtherEmbedding(model_id) - - return model diff --git a/web/src/stores/graphStore.js b/web/src/stores/graphStore.js index aa38968a..0903bd0b 100644 --- a/web/src/stores/graphStore.js +++ b/web/src/stores/graphStore.js @@ -47,19 +47,19 @@ export const useGraphStore = defineStore('graph', { // 首先尝试通过dynamicId匹配 let foundEdge = state.rawGraph.edges.find(edge => edge.dynamicId === state.selectedEdge) if (foundEdge) return foundEdge - + // 如果没找到，尝试通过原始ID匹配 foundEdge = state.rawGraph.edges.find(edge => edge.id === state.selectedEdge) if (foundEdge) return foundEdge - + // 最后尝试通过source->target格式匹配 const [source, target] = state.selectedEdge.split('->') if (source && target) { - foundEdge = state.rawGraph.edges.find(edge => + foundEdge = state.rawGraph.edges.find(edge => edge.source === source.trim() && edge.target === target.trim() ) } - + return foundEdge || null }, @@ -200,7 +200,7 @@ export const useGraphStore = defineStore('graph', { const nodeId = String(node.id) const labels = node.labels || [node.entity_type || 'unknown'] const entityType = node.entity_type || labels[0] || 'unknown' - + const processedNode = { id: nodeId, labels: Array.isArray(labels) ? labels.map(String) : [String(labels)], @@ -246,10 +246,10 @@ export const useGraphStore = defineStore('graph', { const sourceId = String(edge.source) const targetId = String(edge.target) const dynamicId = `${sourceId}-${targetId}-${index}` - + // 适配新的LightRAG API格式 const weight = Number(edge.properties?.weight || edge.weight || 1.0) - + const processedEdge = { id: String(edge.id), source: sourceId, @@ -345,7 +345,7 @@ export const useGraphStore = defineStore('graph', { try { // 使用动态ID作为Sigma边ID，避免重复 const sigmaEdgeId = edge.dynamicId || `${edge.source}->${edge.target}` - + // 检查是否已存在相同的边 if (!sigmaGraph.hasEdge(sigmaEdgeId)) { sigmaGraph.addEdgeWithKey(sigmaEdgeId, String(edge.source), String(edge.target), edgeAttributes) diff --git a/web/src/views/DataBaseInfoView.vue b/web/src/views/DataBaseInfoView.vue index 1865e706..629c99e6 100644 --- a/web/src/views/DataBaseInfoView.vue +++ b/web/src/views/DataBaseInfoView.vue @@ -289,20 +289,6 @@ -
-
-

片段最大Token数：

- -
-
-

关系描述最大Token数：

- -
-
-

实体描述最大Token数：

- -
-
@@ -314,8 +300,8 @@
-
@@ -396,9 +382,6 @@ const meta = reactive({ only_need_context: true, only_need_prompt: false, top_k: 10, - max_token_for_text_unit: 4000, - max_token_for_global_context: 4000, - max_token_for_local_context: 4000, }); const enable_ocr_options = ref([ @@ -1541,10 +1524,10 @@ const toggleAutoRefresh = (checked) => { .knowledge-graph-container { height: calc(100vh - 200px); - + :deep(.knowledge-graph-viewer) { height: 100%; - + .sigma-container { height: calc(100% - 80px); } diff --git a/web/src/views/GraphView.vue b/web/src/views/GraphView.vue index 4f7d4a86..61f0dcc5 100644 --- a/web/src/views/GraphView.vue +++ b/web/src/views/GraphView.vue @@ -231,9 +231,9 @@ const onSearch = () => { // return // } - if (!confirm(`构建图数据库时向量模型为 ${graphInfo?.value?.embed_model_name}，当前向量模型为 ${cur_embed_model.value}，是否继续查询？`)) { - return - } + // if (!confirm(`构建图数据库时向量模型为 ${graphInfo?.value?.embed_model_name}，当前向量模型为 ${cur_embed_model.value}，是否继续查询？`)) { + // return + // } } if (!state.searchInput) {