From 812bb95f8d8b03d640bcc72bcb452398129f2c18 Mon Sep 17 00:00:00 2001 From: Wenjie Zhang Date: Thu, 29 Jan 2026 21:08:43 +0800 Subject: [PATCH] =?UTF-8?q?format:=20=20=E6=A0=BC=E5=BC=8F=E5=8C=96?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/knowledge/indexing.py | 1 + src/knowledge/utils/kb_utils.py | 4 +- src/knowledge/utils/url_fetcher.py | 55 +++++--- src/knowledge/utils/url_validator.py | 1 + uv.lock | 42 ++++++ web/src/stores/info.js | 2 +- web/src/views/HomeView.vue | 196 +++++++++++++-------------- 7 files changed, 176 insertions(+), 125 deletions(-) diff --git a/src/knowledge/indexing.py b/src/knowledge/indexing.py index 6c50ece3..afef2baa 100644 --- a/src/knowledge/indexing.py +++ b/src/knowledge/indexing.py @@ -727,6 +727,7 @@ async def process_url_to_markdown(url: str, params: dict | None = None) -> str: try: import httpx + # 使用异步 HTTP 客户端获取页面 async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client: response = await client.get(url, headers={"User-Agent": "Mozilla/5.0"}) diff --git a/src/knowledge/utils/kb_utils.py b/src/knowledge/utils/kb_utils.py index 370cae1b..30984ff2 100644 --- a/src/knowledge/utils/kb_utils.py +++ b/src/knowledge/utils/kb_utils.py @@ -194,13 +194,13 @@ async def prepare_item_metadata(item: str, content_type: str, db_id: str, params # 使用预处理信息 filename = pre_info.get("filename", item) # 通常是原始 URL - + # 截断文件名以适应数据库限制 (512 chars)，保留部分后缀信息如果可能 if len(filename) > 500: filename_display = filename[:400] + "..." + filename[-90:] else: filename_display = filename - + file_type = "html" # 强制转换为 html 类型，以便后续作为文件处理 item_path = pre_info["path"] # MinIO path content_hash = pre_info["content_hash"] diff --git a/src/knowledge/utils/url_fetcher.py b/src/knowledge/utils/url_fetcher.py index c2021456..1c44b76f 100644 --- a/src/knowledge/utils/url_fetcher.py +++ b/src/knowledge/utils/url_fetcher.py @@ -1,8 +1,10 @@ -import httpx -from urllib.parse import urlparse -import socket import ipaddress -from src.knowledge.utils.url_validator import validate_url, is_url_parsing_enabled +import socket +from urllib.parse import urlparse + +import httpx + +from src.knowledge.utils.url_validator import is_url_parsing_enabled, validate_url from src.utils import logger # 最大允许下载大小 (例如 10MB) @@ -10,9 +12,11 @@ MAX_DOWNLOAD_SIZE = 10 * 1024 * 1024 # 允许的 Content-Type ALLOWED_CONTENT_TYPES = ["text/html", "application/xhtml+xml"] + async def is_private_ip(hostname: str) -> bool: """Check if the hostname resolves to a private IP address.""" import asyncio + try: # Resolve hostname to IP in a separate thread to avoid blocking the event loop ip_list = await asyncio.to_thread(socket.getaddrinfo, hostname, None) @@ -29,6 +33,7 @@ async def is_private_ip(hostname: str) -> bool: # For now, we return False assuming standard DNS failure handling. return False + async def fetch_url_content(url: str, max_size: int = MAX_DOWNLOAD_SIZE) -> tuple[bytes, str]: """ Fetch URL content with security checks (size limit, content type, private IP blocking). @@ -59,68 +64,74 @@ async def fetch_url_content(url: str, max_size: int = MAX_DOWNLOAD_SIZE) -> tupl current_url = url redirect_count = 0 max_redirects = 5 - + # We handle redirects manually to check each target URL against whitelist/private IP try: async with httpx.AsyncClient(timeout=30.0, follow_redirects=False) as client: while True: logger.info(f"Fetching URL: {current_url}") - + # Request headers headers = { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/91.0.4472.124 " + "Safari/537.36" + ) } - + # Stream the response to check headers before downloading body async with client.stream("GET", current_url, headers=headers) as response: # Handle Redirects if response.status_code in (301, 302, 303, 307, 308): if redirect_count >= max_redirects: raise ValueError("Too many redirects") - + redirect_count += 1 location = response.headers.get("Location") if not location: raise ValueError("Redirect response missing Location header") - + # Handle relative redirects if location.startswith("/"): parsed_current = urlparse(current_url) current_url = f"{parsed_current.scheme}://{parsed_current.netloc}{location}" elif not location.startswith("http"): # Handle relative path without /? or other weird cases, or assume absolute - parsed_current = urlparse(current_url) - # simple join - from urllib.parse import urljoin - current_url = urljoin(current_url, location) + parsed_current = urlparse(current_url) + # simple join + from urllib.parse import urljoin + + current_url = urljoin(current_url, location) else: current_url = location - + # Validate the new URL is_valid, error_msg = validate_url(current_url) if not is_valid: raise ValueError(f"Redirected to invalid URL: {error_msg}") - + parsed_new = urlparse(current_url) if await is_private_ip(parsed_new.hostname): raise ValueError("Redirected to private IP address") - - continue # Start new request - + + continue # Start new request + response.raise_for_status() - + # Check Content-Type content_type = response.headers.get("Content-Type", "").lower() if not any(allowed in content_type for allowed in ALLOWED_CONTENT_TYPES): raise ValueError(f"Unsupported Content-Type: {content_type}. Only HTML is supported.") - + # Download content with size limit content = bytearray() async for chunk in response.aiter_bytes(): content.extend(chunk) if len(content) > max_size: raise ValueError(f"Content size exceeds limit of {max_size} bytes") - + return bytes(content), current_url except httpx.HTTPError as e: diff --git a/src/knowledge/utils/url_validator.py b/src/knowledge/utils/url_validator.py index e151d97c..e6109508 100644 --- a/src/knowledge/utils/url_validator.py +++ b/src/knowledge/utils/url_validator.py @@ -1,4 +1,5 @@ """URL validation utilities for whitelist-based URL parsing.""" + import os from urllib.parse import urlparse diff --git a/uv.lock b/uv.lock index 3cce4d34..6908cd70 100644 --- a/uv.lock +++ b/uv.lock @@ -559,6 +559,15 @@ wheels = [ { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e8/cb/2da4cc83f5edb9c3257d09e1e7ab7b23f049c7962cae8d842bbef0a9cec9/cryptography-46.0.3-cp38-abi3-win_arm64.whl", hash = "sha256:d89c3468de4cdc4f08a57e214384d0471911a3830fcdaf7a8cc587e42a866372", size = 2918740, upload-time = "2025-10-15T23:18:12.277Z" }, ] +[[package]] +name = "cssselect" +version = "1.4.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ec/2e/cdfd8b01c37cbf4f9482eefd455853a3cf9c995029a46acd31dfaa9c1dd6/cssselect-1.4.0.tar.gz", hash = "sha256:fdaf0a1425e17dfe8c5cf66191d211b357cf7872ae8afc4c6762ddd8ac47fc92", size = 40589, upload-time = "2026-01-29T07:00:26.701Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/20/0c/7bb51e3acfafd16c48875bf3db03607674df16f5b6ef8d056586af7e2b8b/cssselect-1.4.0-py3-none-any.whl", hash = "sha256:c0ec5c0191c8ee39fcc8afc1540331d8b55b0183478c50e9c8a79d44dbceb1d8", size = 18540, upload-time = "2026-01-29T07:00:24.994Z" }, +] + [[package]] name = "dashscope" version = "1.25.8" @@ -2109,6 +2118,23 @@ wheels = [ { url = "https://pypi.tuna.tsinghua.edu.cn/packages/31/ef/dcf1d29c3f530577f61e5fe2f1bd72929acf779953668a8a47a479ae6f26/lxml-6.0.2-cp313-cp313-win_arm64.whl", hash = "sha256:13dcecc9946dca97b11b7c40d29fba63b55ab4170d3c0cf8c0c164343b9bfdcf", size = 3671248, upload-time = "2025-09-22T04:02:27.918Z" }, ] +[package.optional-dependencies] +html-clean = [ + { name = "lxml-html-clean" }, +] + +[[package]] +name = "lxml-html-clean" +version = "0.4.3" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "lxml" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d9/cb/c9c5bb2a9c47292e236a808dd233a03531f53b626f36259dcd32b49c76da/lxml_html_clean-0.4.3.tar.gz", hash = "sha256:c9df91925b00f836c807beab127aac82575110eacff54d0a75187914f1bd9d8c", size = 21498, upload-time = "2025-10-02T20:49:24.895Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/10/4a/63a9540e3ca73709f4200564a737d63a4c8c9c4dd032bab8535f507c190a/lxml_html_clean-0.4.3-py3-none-any.whl", hash = "sha256:63fd7b0b9c3a2e4176611c2ca5d61c4c07ffca2de76c14059a81a2825833731e", size = 14177, upload-time = "2025-10-02T20:49:23.749Z" }, +] + [[package]] name = "markdown-it-py" version = "4.0.0" @@ -3603,6 +3629,20 @@ wheels = [ { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ba/12/1e5497183bdbe782dbb91bad1d0d2297dba4d2831b2652657f7517bfc6df/rapidocr_onnxruntime-1.4.4-py3-none-any.whl", hash = "sha256:971d7d5f223a7a808662229df1ef69893809d8457d834e6373d3854bc1782cbf", size = 14915192, upload-time = "2025-01-17T01:48:25.104Z" }, ] +[[package]] +name = "readability-lxml" +version = "0.8.4.1" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "chardet" }, + { name = "cssselect" }, + { name = "lxml", extra = ["html-clean"] }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/55/3e/dc87d97532ddad58af786ec89c7036182e352574c1cba37bf2bf783d2b15/readability_lxml-0.8.4.1.tar.gz", hash = "sha256:9d2924f5942dd7f37fb4da353263b22a3e877ccf922d0e45e348e4177b035a53", size = 22874, upload-time = "2025-05-03T21:11:45.493Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c7/75/2cc58965097e351415af420be81c4665cf80da52a17ef43c01ffbe2caf91/readability_lxml-0.8.4.1-py3-none-any.whl", hash = "sha256:874c0cea22c3bf2b78c7f8df831bfaad3c0a89b7301d45a188db581652b4b465", size = 19912, upload-time = "2025-05-03T21:11:43.993Z" }, +] + [[package]] name = "referencing" version = "0.37.0" @@ -4956,6 +4996,7 @@ dependencies = [ { name = "python-multipart" }, { name = "pyyaml" }, { name = "rapidocr-onnxruntime" }, + { name = "readability-lxml" }, { name = "rich" }, { name = "sqlalchemy", extra = ["asyncio"] }, { name = "tabulate" }, @@ -5033,6 +5074,7 @@ requires-dist = [ { name = "python-multipart", specifier = ">=0.0.20" }, { name = "pyyaml", specifier = ">=6.0.2" }, { name = "rapidocr-onnxruntime", specifier = ">=1.4.4" }, + { name = "readability-lxml", specifier = ">=0.8.1" }, { name = "rich", specifier = ">=13.7.1" }, { name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.0.0" }, { name = "tabulate", specifier = ">=0.9.0" }, diff --git a/web/src/stores/info.js b/web/src/stores/info.js index 68b92769..69e6ba8b 100644 --- a/web/src/stores/info.js +++ b/web/src/stores/info.js @@ -8,7 +8,7 @@ export const useInfoStore = defineStore('info', () => { const isLoading = ref(false) const isLoaded = ref(false) const debugMode = ref(false) - const error = ref(null) // 错误信息 + const error = ref(null) // 错误信息 // 计算属性 - 组织信息 const organization = computed( diff --git a/web/src/views/HomeView.vue b/web/src/views/HomeView.vue index 055e816f..88fd7622 100644 --- a/web/src/views/HomeView.vue +++ b/web/src/views/HomeView.vue @@ -8,11 +8,7 @@
- + @@ -22,111 +18,111 @@