feat(channel/utils): 新增工具模块与文件服务类
新增了通用工具包,包含文件操作工具、延迟加载工具类与函数,以及文件服务类,提供文件分类、安全命名与上传保存功能
This commit is contained in:
parent
849d21120f
commit
3b487fae57
35
backend/package/yuxi/channel/utils/__init__.py
Normal file
35
backend/package/yuxi/channel/utils/__init__.py
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
from yuxi.channel.utils.files import (
|
||||||
|
compute_file_hash,
|
||||||
|
delete_file,
|
||||||
|
ensure_directory,
|
||||||
|
list_files,
|
||||||
|
read_json_file,
|
||||||
|
safe_write_file,
|
||||||
|
write_json_file,
|
||||||
|
)
|
||||||
|
from yuxi.channel.utils.lazy import (
|
||||||
|
LRULazyModule,
|
||||||
|
Lazy,
|
||||||
|
LazyClass,
|
||||||
|
LazyModule,
|
||||||
|
lazy_property,
|
||||||
|
run_sync,
|
||||||
|
run_sync_partial,
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"Lazy",
|
||||||
|
"lazy_property",
|
||||||
|
"LazyModule",
|
||||||
|
"LazyClass",
|
||||||
|
"run_sync",
|
||||||
|
"run_sync_partial",
|
||||||
|
"LRULazyModule",
|
||||||
|
"read_json_file",
|
||||||
|
"write_json_file",
|
||||||
|
"compute_file_hash",
|
||||||
|
"ensure_directory",
|
||||||
|
"safe_write_file",
|
||||||
|
"list_files",
|
||||||
|
"delete_file",
|
||||||
|
]
|
||||||
42
backend/package/yuxi/channel/utils/file_service.py
Normal file
42
backend/package/yuxi/channel/utils/file_service.py
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg"}
|
||||||
|
VIDEO_EXTENSIONS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".flv", ".wmv"}
|
||||||
|
ALLOWED_EXTENSIONS = IMAGE_EXTENSIONS | VIDEO_EXTENSIONS | {".pdf", ".txt", ".json", ".zip", ".rar"}
|
||||||
|
|
||||||
|
_DEFAULT_MAX_FILE_BYTES = 50 * 1024 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
class FileService:
|
||||||
|
def __init__(self, storage_dir: str | Path, max_file_bytes: int = _DEFAULT_MAX_FILE_BYTES):
|
||||||
|
self.storage_dir = Path(storage_dir)
|
||||||
|
self.storage_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
self.max_file_bytes = max_file_bytes
|
||||||
|
|
||||||
|
def classify(self, filename: str) -> str:
|
||||||
|
ext = os.path.splitext(filename)[1].lower()
|
||||||
|
if ext in IMAGE_EXTENSIONS:
|
||||||
|
return "image"
|
||||||
|
if ext in VIDEO_EXTENSIONS:
|
||||||
|
return "video"
|
||||||
|
return "file"
|
||||||
|
|
||||||
|
def generate_safe_name(self, original_name: str) -> str:
|
||||||
|
name = self._sanitize_filename(original_name)
|
||||||
|
h = hashlib.sha256(name.encode()).hexdigest()[:12]
|
||||||
|
ext = os.path.splitext(original_name)[1].lower()
|
||||||
|
return f"{h}_{name}"
|
||||||
|
|
||||||
|
async def save_upload(self, content: bytes, safe_name: str) -> Path:
|
||||||
|
dest = self.storage_dir / safe_name
|
||||||
|
dest.write_bytes(content)
|
||||||
|
return dest
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _sanitize_filename(filename: str) -> str:
|
||||||
|
return re.sub(r"[^\w.\-]", "_", filename)
|
||||||
78
backend/package/yuxi/channel/utils/files.py
Normal file
78
backend/package/yuxi/channel/utils/files.py
Normal file
@ -0,0 +1,78 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def read_json_file(file_path: str | Path) -> dict[str, Any]:
|
||||||
|
"""读取 JSON 文件并返回字典。"""
|
||||||
|
file_path = Path(file_path)
|
||||||
|
if not file_path.exists():
|
||||||
|
raise FileNotFoundError(f"File not found: {file_path}")
|
||||||
|
return json.loads(file_path.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
def write_json_file(file_path: str | Path, data: dict[str, Any], *, indent: int = 2) -> None:
|
||||||
|
"""将字典写入 JSON 文件。"""
|
||||||
|
file_path = Path(file_path)
|
||||||
|
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
file_path.write_text(json.dumps(data, indent=indent, ensure_ascii=False), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def compute_file_hash(file_path: str | Path, algorithm: str = "sha256") -> str:
|
||||||
|
"""计算文件的哈希值。"""
|
||||||
|
file_path = Path(file_path)
|
||||||
|
if not file_path.exists():
|
||||||
|
raise FileNotFoundError(f"File not found: {file_path}")
|
||||||
|
|
||||||
|
hasher = hashlib.new(algorithm)
|
||||||
|
hasher.update(file_path.read_bytes())
|
||||||
|
return hasher.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_directory(dir_path: str | Path) -> Path:
|
||||||
|
"""确保目录存在,如果不存在则创建。"""
|
||||||
|
path = Path(dir_path)
|
||||||
|
path.mkdir(parents=True, exist_ok=True)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def safe_write_file(file_path: str | Path, content: str, *, encoding: str = "utf-8") -> None:
|
||||||
|
"""安全写入文件:先写入临时文件,再原子替换。"""
|
||||||
|
file_path = Path(file_path)
|
||||||
|
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
temp_path = file_path.with_suffix(f"{file_path.suffix}.tmp")
|
||||||
|
temp_path.write_text(content, encoding=encoding)
|
||||||
|
temp_path.replace(file_path)
|
||||||
|
|
||||||
|
|
||||||
|
def list_files(
|
||||||
|
directory: str | Path,
|
||||||
|
pattern: str = "*",
|
||||||
|
*,
|
||||||
|
recursive: bool = False,
|
||||||
|
) -> list[Path]:
|
||||||
|
"""列出目录中的文件。"""
|
||||||
|
directory = Path(directory)
|
||||||
|
if not directory.exists():
|
||||||
|
return []
|
||||||
|
|
||||||
|
if recursive:
|
||||||
|
return list(directory.rglob(pattern))
|
||||||
|
return list(directory.glob(pattern))
|
||||||
|
|
||||||
|
|
||||||
|
def delete_file(file_path: str | Path, *, missing_ok: bool = True) -> bool:
|
||||||
|
"""删除文件。"""
|
||||||
|
file_path = Path(file_path)
|
||||||
|
try:
|
||||||
|
file_path.unlink(missing_ok=missing_ok)
|
||||||
|
return True
|
||||||
|
except OSError:
|
||||||
|
logger.warning("Failed to delete file: %s", file_path)
|
||||||
|
return False
|
||||||
195
backend/package/yuxi/channel/utils/lazy.py
Normal file
195
backend/package/yuxi/channel/utils/lazy.py
Normal file
@ -0,0 +1,195 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import functools
|
||||||
|
import importlib
|
||||||
|
import logging
|
||||||
|
import threading
|
||||||
|
from collections import OrderedDict
|
||||||
|
from typing import Any, Callable, Generic, TypeVar
|
||||||
|
|
||||||
|
T = TypeVar("T")
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class Lazy(Generic[T]):
|
||||||
|
"""延迟初始化包装器 — 线程安全,支持自定义工厂函数。"""
|
||||||
|
|
||||||
|
def __init__(self, factory: Callable[[], T]):
|
||||||
|
self._factory = factory
|
||||||
|
self._value: T | None = None
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._initialized = False
|
||||||
|
|
||||||
|
@property
|
||||||
|
def value(self) -> T:
|
||||||
|
if not self._initialized:
|
||||||
|
with self._lock:
|
||||||
|
if not self._initialized:
|
||||||
|
self._value = self._factory()
|
||||||
|
self._initialized = True
|
||||||
|
return self._value
|
||||||
|
|
||||||
|
def reset(self) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self._value = None
|
||||||
|
self._initialized = False
|
||||||
|
|
||||||
|
def is_initialized(self) -> bool:
|
||||||
|
return self._initialized
|
||||||
|
|
||||||
|
|
||||||
|
def lazy_property(factory: Callable[[], T]) -> property:
|
||||||
|
"""装饰器:将方法转换为延迟加载属性。"""
|
||||||
|
_lazy = Lazy(factory)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def prop(self) -> T:
|
||||||
|
return _lazy.value
|
||||||
|
|
||||||
|
return prop
|
||||||
|
|
||||||
|
|
||||||
|
class LazyModule:
|
||||||
|
"""延迟导入模块 — 避免循环导入与启动时性能开销。"""
|
||||||
|
|
||||||
|
def __init__(self, module_name: str):
|
||||||
|
self._module_name = module_name
|
||||||
|
self._module: Any = None
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
|
||||||
|
def __getattr__(self, name: str) -> Any:
|
||||||
|
if self._module is None:
|
||||||
|
with self._lock:
|
||||||
|
if self._module is None:
|
||||||
|
self._module = importlib.import_module(self._module_name)
|
||||||
|
return getattr(self._module, name)
|
||||||
|
|
||||||
|
def __call__(self, *args: Any, **kwargs: Any) -> Any:
|
||||||
|
return self.__getattr__("__call__")(*args, **kwargs)
|
||||||
|
|
||||||
|
def __dir__(self) -> list[str]:
|
||||||
|
return dir(self._module if self._module is not None else importlib.import_module(self._module_name))
|
||||||
|
|
||||||
|
def __repr__(self) -> str:
|
||||||
|
status = "loaded" if self._module is not None else "not loaded"
|
||||||
|
return f"LazyModule({self._module_name!r}, {status})"
|
||||||
|
|
||||||
|
|
||||||
|
class LazyClass:
|
||||||
|
"""延迟导入类 — 按需解析模块中的类,支持实例化和属性访问。"""
|
||||||
|
|
||||||
|
def __init__(self, module_name: str, class_name: str):
|
||||||
|
self._module_name = module_name
|
||||||
|
self._class_name = class_name
|
||||||
|
self._cls: type | None = None
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
|
||||||
|
def _resolve(self) -> type:
|
||||||
|
if self._cls is None:
|
||||||
|
with self._lock:
|
||||||
|
if self._cls is None:
|
||||||
|
module = importlib.import_module(self._module_name)
|
||||||
|
self._cls = getattr(module, self._class_name)
|
||||||
|
return self._cls
|
||||||
|
|
||||||
|
def __call__(self, *args: Any, **kwargs: Any) -> Any:
|
||||||
|
return self._resolve()(*args, **kwargs)
|
||||||
|
|
||||||
|
def __getattr__(self, name: str) -> Any:
|
||||||
|
return getattr(self._resolve(), name)
|
||||||
|
|
||||||
|
def __repr__(self) -> str:
|
||||||
|
status = "resolved" if self._cls is not None else "not resolved"
|
||||||
|
return f"LazyClass({self._module_name!r}, {self._class_name!r}, {status})"
|
||||||
|
|
||||||
|
|
||||||
|
async def run_sync(func: Callable[..., T], *args: Any, **kwargs: Any) -> T:
|
||||||
|
"""在默认线程池中运行同步函数,返回协程。"""
|
||||||
|
return await asyncio.get_running_loop().run_in_executor(
|
||||||
|
None, functools.partial(func, *args, **kwargs)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def run_sync_partial(func: Callable[..., T], *args: Any, **kwargs: Any) -> T:
|
||||||
|
"""带偏应用的 run_sync — 将 *args/**kwargs 与 func 绑定后在线程池执行。"""
|
||||||
|
bound = functools.partial(func, *args, **kwargs)
|
||||||
|
return await asyncio.get_running_loop().run_in_executor(None, bound)
|
||||||
|
|
||||||
|
|
||||||
|
class LRULazyModule:
|
||||||
|
"""LRU 缓存的延迟模块加载器 — 线程安全,支持异步 get/invalidate/clear。"""
|
||||||
|
|
||||||
|
def __init__(self, maxsize: int):
|
||||||
|
if not isinstance(maxsize, int) or maxsize <= 0:
|
||||||
|
raise ValueError("maxsize must be a positive integer")
|
||||||
|
self._maxsize = maxsize
|
||||||
|
self._cache: OrderedDict[str, Any] = OrderedDict()
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._pending: dict[str, asyncio.Event] = {}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def size(self) -> int:
|
||||||
|
with self._lock:
|
||||||
|
return len(self._cache)
|
||||||
|
|
||||||
|
def __len__(self) -> int:
|
||||||
|
return self.size
|
||||||
|
|
||||||
|
def __contains__(self, key: str) -> bool:
|
||||||
|
with self._lock:
|
||||||
|
return key in self._cache
|
||||||
|
|
||||||
|
def __getitem__(self, key: str) -> Any:
|
||||||
|
with self._lock:
|
||||||
|
if key not in self._cache:
|
||||||
|
raise KeyError(key)
|
||||||
|
self._cache.move_to_end(key)
|
||||||
|
return self._cache[key]
|
||||||
|
|
||||||
|
async def get(self, module_name: str) -> Any:
|
||||||
|
with self._lock:
|
||||||
|
if module_name in self._cache:
|
||||||
|
self._cache.move_to_end(module_name)
|
||||||
|
return self._cache[module_name]
|
||||||
|
|
||||||
|
if module_name in self._pending:
|
||||||
|
event = self._pending[module_name]
|
||||||
|
is_loader = False
|
||||||
|
else:
|
||||||
|
event = asyncio.Event()
|
||||||
|
self._pending[module_name] = event
|
||||||
|
is_loader = True
|
||||||
|
|
||||||
|
if not is_loader:
|
||||||
|
await event.wait()
|
||||||
|
with self._lock:
|
||||||
|
return self._cache[module_name]
|
||||||
|
|
||||||
|
try:
|
||||||
|
module = importlib.import_module(module_name)
|
||||||
|
except Exception:
|
||||||
|
with self._lock:
|
||||||
|
self._pending.pop(module_name, None)
|
||||||
|
event.set()
|
||||||
|
raise
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
self._pending.pop(module_name, None)
|
||||||
|
self._cache[module_name] = module
|
||||||
|
self._cache.move_to_end(module_name)
|
||||||
|
if len(self._cache) > self._maxsize:
|
||||||
|
self._cache.popitem(last=False)
|
||||||
|
event.set()
|
||||||
|
return module
|
||||||
|
|
||||||
|
async def invalidate(self, module_name: str) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self._cache.pop(module_name, None)
|
||||||
|
|
||||||
|
async def clear(self) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self._cache.clear()
|
||||||
|
|
||||||
|
def __repr__(self) -> str:
|
||||||
|
return f"LRULazyModule(maxsize={self._maxsize}, size={self.size})"
|
||||||
Loading…
Reference in New Issue
Block a user