format: 格式化代码

This commit is contained in:
Wenjie Zhang 2026-01-29 21:08:43 +08:00
parent 5a376dd1b9
commit 812bb95f8d
7 changed files with 176 additions and 125 deletions

View File

@ -727,6 +727,7 @@ async def process_url_to_markdown(url: str, params: dict | None = None) -> str:
try:
import httpx
# 使用异步 HTTP 客户端获取页面
async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
response = await client.get(url, headers={"User-Agent": "Mozilla/5.0"})

View File

@ -194,13 +194,13 @@ async def prepare_item_metadata(item: str, content_type: str, db_id: str, params
# 使用预处理信息
filename = pre_info.get("filename", item) # 通常是原始 URL
# 截断文件名以适应数据库限制 (512 chars),保留部分后缀信息如果可能
if len(filename) > 500:
filename_display = filename[:400] + "..." + filename[-90:]
else:
filename_display = filename
file_type = "html" # 强制转换为 html 类型,以便后续作为文件处理
item_path = pre_info["path"] # MinIO path
content_hash = pre_info["content_hash"]

View File

@ -1,8 +1,10 @@
import httpx
from urllib.parse import urlparse
import socket
import ipaddress
from src.knowledge.utils.url_validator import validate_url, is_url_parsing_enabled
import socket
from urllib.parse import urlparse
import httpx
from src.knowledge.utils.url_validator import is_url_parsing_enabled, validate_url
from src.utils import logger
# 最大允许下载大小 (例如 10MB)
@ -10,9 +12,11 @@ MAX_DOWNLOAD_SIZE = 10 * 1024 * 1024
# 允许的 Content-Type
ALLOWED_CONTENT_TYPES = ["text/html", "application/xhtml+xml"]
async def is_private_ip(hostname: str) -> bool:
"""Check if the hostname resolves to a private IP address."""
import asyncio
try:
# Resolve hostname to IP in a separate thread to avoid blocking the event loop
ip_list = await asyncio.to_thread(socket.getaddrinfo, hostname, None)
@ -29,6 +33,7 @@ async def is_private_ip(hostname: str) -> bool:
# For now, we return False assuming standard DNS failure handling.
return False
async def fetch_url_content(url: str, max_size: int = MAX_DOWNLOAD_SIZE) -> tuple[bytes, str]:
"""
Fetch URL content with security checks (size limit, content type, private IP blocking).
@ -59,68 +64,74 @@ async def fetch_url_content(url: str, max_size: int = MAX_DOWNLOAD_SIZE) -> tupl
current_url = url
redirect_count = 0
max_redirects = 5
# We handle redirects manually to check each target URL against whitelist/private IP
try:
async with httpx.AsyncClient(timeout=30.0, follow_redirects=False) as client:
while True:
logger.info(f"Fetching URL: {current_url}")
# Request headers
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/91.0.4472.124 "
"Safari/537.36"
)
}
# Stream the response to check headers before downloading body
async with client.stream("GET", current_url, headers=headers) as response:
# Handle Redirects
if response.status_code in (301, 302, 303, 307, 308):
if redirect_count >= max_redirects:
raise ValueError("Too many redirects")
redirect_count += 1
location = response.headers.get("Location")
if not location:
raise ValueError("Redirect response missing Location header")
# Handle relative redirects
if location.startswith("/"):
parsed_current = urlparse(current_url)
current_url = f"{parsed_current.scheme}://{parsed_current.netloc}{location}"
elif not location.startswith("http"):
# Handle relative path without /? or other weird cases, or assume absolute
parsed_current = urlparse(current_url)
# simple join
from urllib.parse import urljoin
current_url = urljoin(current_url, location)
parsed_current = urlparse(current_url)
# simple join
from urllib.parse import urljoin
current_url = urljoin(current_url, location)
else:
current_url = location
# Validate the new URL
is_valid, error_msg = validate_url(current_url)
if not is_valid:
raise ValueError(f"Redirected to invalid URL: {error_msg}")
parsed_new = urlparse(current_url)
if await is_private_ip(parsed_new.hostname):
raise ValueError("Redirected to private IP address")
continue # Start new request
continue # Start new request
response.raise_for_status()
# Check Content-Type
content_type = response.headers.get("Content-Type", "").lower()
if not any(allowed in content_type for allowed in ALLOWED_CONTENT_TYPES):
raise ValueError(f"Unsupported Content-Type: {content_type}. Only HTML is supported.")
# Download content with size limit
content = bytearray()
async for chunk in response.aiter_bytes():
content.extend(chunk)
if len(content) > max_size:
raise ValueError(f"Content size exceeds limit of {max_size} bytes")
return bytes(content), current_url
except httpx.HTTPError as e:

View File

@ -1,4 +1,5 @@
"""URL validation utilities for whitelist-based URL parsing."""
import os
from urllib.parse import urlparse

42
uv.lock
View File

@ -559,6 +559,15 @@ wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/e8/cb/2da4cc83f5edb9c3257d09e1e7ab7b23f049c7962cae8d842bbef0a9cec9/cryptography-46.0.3-cp38-abi3-win_arm64.whl", hash = "sha256:d89c3468de4cdc4f08a57e214384d0471911a3830fcdaf7a8cc587e42a866372", size = 2918740, upload-time = "2025-10-15T23:18:12.277Z" },
]
[[package]]
name = "cssselect"
version = "1.4.0"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ec/2e/cdfd8b01c37cbf4f9482eefd455853a3cf9c995029a46acd31dfaa9c1dd6/cssselect-1.4.0.tar.gz", hash = "sha256:fdaf0a1425e17dfe8c5cf66191d211b357cf7872ae8afc4c6762ddd8ac47fc92", size = 40589, upload-time = "2026-01-29T07:00:26.701Z" }
wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/20/0c/7bb51e3acfafd16c48875bf3db03607674df16f5b6ef8d056586af7e2b8b/cssselect-1.4.0-py3-none-any.whl", hash = "sha256:c0ec5c0191c8ee39fcc8afc1540331d8b55b0183478c50e9c8a79d44dbceb1d8", size = 18540, upload-time = "2026-01-29T07:00:24.994Z" },
]
[[package]]
name = "dashscope"
version = "1.25.8"
@ -2109,6 +2118,23 @@ wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/31/ef/dcf1d29c3f530577f61e5fe2f1bd72929acf779953668a8a47a479ae6f26/lxml-6.0.2-cp313-cp313-win_arm64.whl", hash = "sha256:13dcecc9946dca97b11b7c40d29fba63b55ab4170d3c0cf8c0c164343b9bfdcf", size = 3671248, upload-time = "2025-09-22T04:02:27.918Z" },
]
[package.optional-dependencies]
html-clean = [
{ name = "lxml-html-clean" },
]
[[package]]
name = "lxml-html-clean"
version = "0.4.3"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
dependencies = [
{ name = "lxml" },
]
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d9/cb/c9c5bb2a9c47292e236a808dd233a03531f53b626f36259dcd32b49c76da/lxml_html_clean-0.4.3.tar.gz", hash = "sha256:c9df91925b00f836c807beab127aac82575110eacff54d0a75187914f1bd9d8c", size = 21498, upload-time = "2025-10-02T20:49:24.895Z" }
wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/10/4a/63a9540e3ca73709f4200564a737d63a4c8c9c4dd032bab8535f507c190a/lxml_html_clean-0.4.3-py3-none-any.whl", hash = "sha256:63fd7b0b9c3a2e4176611c2ca5d61c4c07ffca2de76c14059a81a2825833731e", size = 14177, upload-time = "2025-10-02T20:49:23.749Z" },
]
[[package]]
name = "markdown-it-py"
version = "4.0.0"
@ -3603,6 +3629,20 @@ wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/ba/12/1e5497183bdbe782dbb91bad1d0d2297dba4d2831b2652657f7517bfc6df/rapidocr_onnxruntime-1.4.4-py3-none-any.whl", hash = "sha256:971d7d5f223a7a808662229df1ef69893809d8457d834e6373d3854bc1782cbf", size = 14915192, upload-time = "2025-01-17T01:48:25.104Z" },
]
[[package]]
name = "readability-lxml"
version = "0.8.4.1"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
dependencies = [
{ name = "chardet" },
{ name = "cssselect" },
{ name = "lxml", extra = ["html-clean"] },
]
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/55/3e/dc87d97532ddad58af786ec89c7036182e352574c1cba37bf2bf783d2b15/readability_lxml-0.8.4.1.tar.gz", hash = "sha256:9d2924f5942dd7f37fb4da353263b22a3e877ccf922d0e45e348e4177b035a53", size = 22874, upload-time = "2025-05-03T21:11:45.493Z" }
wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/c7/75/2cc58965097e351415af420be81c4665cf80da52a17ef43c01ffbe2caf91/readability_lxml-0.8.4.1-py3-none-any.whl", hash = "sha256:874c0cea22c3bf2b78c7f8df831bfaad3c0a89b7301d45a188db581652b4b465", size = 19912, upload-time = "2025-05-03T21:11:43.993Z" },
]
[[package]]
name = "referencing"
version = "0.37.0"
@ -4956,6 +4996,7 @@ dependencies = [
{ name = "python-multipart" },
{ name = "pyyaml" },
{ name = "rapidocr-onnxruntime" },
{ name = "readability-lxml" },
{ name = "rich" },
{ name = "sqlalchemy", extra = ["asyncio"] },
{ name = "tabulate" },
@ -5033,6 +5074,7 @@ requires-dist = [
{ name = "python-multipart", specifier = ">=0.0.20" },
{ name = "pyyaml", specifier = ">=6.0.2" },
{ name = "rapidocr-onnxruntime", specifier = ">=1.4.4" },
{ name = "readability-lxml", specifier = ">=0.8.1" },
{ name = "rich", specifier = ">=13.7.1" },
{ name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.0.0" },
{ name = "tabulate", specifier = ">=0.9.0" },

View File

@ -8,7 +8,7 @@ export const useInfoStore = defineStore('info', () => {
const isLoading = ref(false)
const isLoaded = ref(false)
const debugMode = ref(false)
const error = ref(null) // 错误信息
const error = ref(null) // 错误信息
// 计算属性 - 组织信息
const organization = computed(

View File

@ -8,11 +8,7 @@
<!-- 错误状态 -->
<div v-else-if="error" class="error-container">
<a-result
status="error"
:title="error.title"
:sub-title="error.message"
>
<a-result status="error" :title="error.title" :sub-title="error.message">
<template #extra>
<a-button type="primary" @click="retryLoad">重试</a-button>
</template>
@ -22,111 +18,111 @@
<!-- 正常内容 -->
<template v-else>
<div class="hero-section">
<div class="glass-header">
<div class="logo">
<img
:src="infoStore.organization.logo"
:alt="infoStore.organization.name"
class="logo-img"
/>
<span class="logo-text">{{ infoStore.organization.name }}</span>
</div>
<nav class="nav-links">
<router-link
to="/agent"
class="nav-link"
v-if="userStore.isLoggedIn && userStore.isAdmin"
>
<span>智能体</span>
</router-link>
<router-link
to="/graph"
class="nav-link"
v-if="userStore.isLoggedIn && userStore.isAdmin"
>
<span>知识图谱</span>
</router-link>
<router-link
to="/database"
class="nav-link"
v-if="userStore.isLoggedIn && userStore.isAdmin"
>
<span>知识库</span>
</router-link>
</nav>
<div class="header-actions">
<div class="github-link">
<a href="https://github.com/xerrors/Yuxi-Know" target="_blank">
<svg height="20" width="20" viewBox="0 0 16 16" version="1.1">
<path
fill-rule="evenodd"
d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"
></path>
</svg>
</a>
<div class="glass-header">
<div class="logo">
<img
:src="infoStore.organization.logo"
:alt="infoStore.organization.name"
class="logo-img"
/>
<span class="logo-text">{{ infoStore.organization.name }}</span>
</div>
<UserInfoComponent :show-button="true" />
</div>
</div>
<div class="hero-layout">
<div class="hero-content">
<h1 class="title">{{ infoStore.branding.title }}</h1>
<p class="subtitle">{{ infoStore.branding.subtitle }}</p>
<!-- <p class="description">{{ infoStore.branding.description }}</p> -->
<div class="hero-actions">
<button class="button-base primary" @click="goToChat">开始对话</button>
<a
class="button-base secondary"
href="https://xerrors.github.io/Yuxi-Know/"
target="_blank"
>查看文档</a
<nav class="nav-links">
<router-link
to="/agent"
class="nav-link"
v-if="userStore.isLoggedIn && userStore.isAdmin"
>
</div>
</div>
<div class="insight-panel" v-if="featureCards.length">
<div class="stat-card" v-for="card in featureCards" :key="card.label">
<div class="stat-headline">
<span class="stat-icon" v-if="card.icon">
<component :is="card.icon" />
</span>
<p class="stat-value">{{ card.value }}</p>
<span>智能体</span>
</router-link>
<router-link
to="/graph"
class="nav-link"
v-if="userStore.isLoggedIn && userStore.isAdmin"
>
<span>知识图谱</span>
</router-link>
<router-link
to="/database"
class="nav-link"
v-if="userStore.isLoggedIn && userStore.isAdmin"
>
<span>知识库</span>
</router-link>
</nav>
<div class="header-actions">
<div class="github-link">
<a href="https://github.com/xerrors/Yuxi-Know" target="_blank">
<svg height="20" width="20" viewBox="0 0 16 16" version="1.1">
<path
fill-rule="evenodd"
d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"
></path>
</svg>
</a>
</div>
<UserInfoComponent :show-button="true" />
</div>
</div>
<div class="hero-layout">
<div class="hero-content">
<h1 class="title">{{ infoStore.branding.title }}</h1>
<p class="subtitle">{{ infoStore.branding.subtitle }}</p>
<!-- <p class="description">{{ infoStore.branding.description }}</p> -->
<div class="hero-actions">
<button class="button-base primary" @click="goToChat">开始对话</button>
<a
class="button-base secondary"
href="https://xerrors.github.io/Yuxi-Know/"
target="_blank"
>查看文档</a
>
</div>
</div>
<div class="insight-panel" v-if="featureCards.length">
<div class="stat-card" v-for="card in featureCards" :key="card.label">
<div class="stat-headline">
<span class="stat-icon" v-if="card.icon">
<component :is="card.icon" />
</span>
<p class="stat-value">{{ card.value }}</p>
</div>
<p class="stat-label">{{ card.label }}</p>
<p class="stat-description">{{ card.description }}</p>
</div>
<p class="stat-label">{{ card.label }}</p>
<p class="stat-description">{{ card.description }}</p>
</div>
</div>
</div>
</div>
<div class="section action-section" v-if="actionLinks.length">
<div class="action-grid">
<a
v-for="action in actionLinks"
:key="action.name"
class="action-card"
:href="action.url"
target="_blank"
rel="noopener noreferrer"
>
<span class="action-icon" v-if="action.icon">
<component :is="action.icon" />
</span>
<div class="action-meta">
<p class="action-title">{{ action.name }}</p>
<p class="action-url">{{ action.url }}</p>
</div>
</a>
<div class="section action-section" v-if="actionLinks.length">
<div class="action-grid">
<a
v-for="action in actionLinks"
:key="action.name"
class="action-card"
:href="action.url"
target="_blank"
rel="noopener noreferrer"
>
<span class="action-icon" v-if="action.icon">
<component :is="action.icon" />
</span>
<div class="action-meta">
<p class="action-title">{{ action.name }}</p>
<p class="action-url">{{ action.url }}</p>
</div>
</a>
</div>
</div>
</div>
<ProjectOverview />
<ProjectOverview />
<footer class="footer">
<div class="footer-content">
<p class="copyright">{{ infoStore.footer?.copyright || '© 2025 All rights reserved' }}</p>
</div>
</footer>
<footer class="footer">
<div class="footer-content">
<p class="copyright">{{ infoStore.footer?.copyright || '© 2025 All rights reserved' }}</p>
</div>
</footer>
</template>
</div>
</template>