410 lines
12 KiB
Python
410 lines
12 KiB
Python
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import signal
|
|
import sys
|
|
import time
|
|
from collections import deque
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from yuxi.channel.security.log_sanitizer import sanitize_text
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DEFAULT_RING_CAPACITY = 1000
|
|
DEFAULT_QUERY_LIMIT = 50
|
|
MAX_BUNDLE_BYTES = 5 * 1024 * 1024
|
|
DEFAULT_BUNDLE_RETENTION = 20
|
|
MAX_SAFE_ERROR_MESSAGE_LENGTH = 500
|
|
|
|
SAFE_REASON_CODE = re.compile(r"^[A-Za-z0-9_.:-]{1,120}$")
|
|
REDACTED_HOSTNAME = "<redacted-hostname>"
|
|
|
|
_PEM_PRIVATE_KEY_PATTERN = re.compile(
|
|
r"-----BEGIN\s+(?:RSA\s+)?PRIVATE\s+KEY-----[\s\S]*?"
|
|
r"-----END\s+(?:RSA\s+)?PRIVATE\s+KEY-----",
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class StabilityEvent:
|
|
seq: int
|
|
ts: float
|
|
type: str
|
|
data: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
class StabilityRingBuffer:
|
|
def __init__(self, capacity: int = DEFAULT_RING_CAPACITY):
|
|
self._buffer: deque[StabilityEvent] = deque(maxlen=capacity)
|
|
self._seq = 0
|
|
self._dropped = 0
|
|
|
|
def record(self, event_type: str, **data: Any) -> None:
|
|
self._seq += 1
|
|
event = StabilityEvent(
|
|
seq=self._seq,
|
|
ts=time.time(),
|
|
type=event_type,
|
|
data=data,
|
|
)
|
|
if len(self._buffer) >= self._buffer.maxlen:
|
|
self._dropped += 1
|
|
self._buffer.append(event)
|
|
|
|
def query(
|
|
self,
|
|
limit: int = DEFAULT_QUERY_LIMIT,
|
|
event_type: str | None = None,
|
|
since_seq: int | None = None,
|
|
) -> list[dict]:
|
|
events = list(self._buffer)
|
|
if event_type is not None:
|
|
events = [e for e in events if e.type == event_type]
|
|
if since_seq is not None:
|
|
events = [e for e in events if e.seq > since_seq]
|
|
events = events[-limit:]
|
|
return [
|
|
{
|
|
"seq": e.seq,
|
|
"ts": e.ts,
|
|
"type": e.type,
|
|
**e.data,
|
|
}
|
|
for e in events
|
|
]
|
|
|
|
def snapshot(
|
|
self,
|
|
limit: int = DEFAULT_QUERY_LIMIT,
|
|
event_type: str | None = None,
|
|
since_seq: int | None = None,
|
|
) -> dict:
|
|
all_events = list(self._buffer)
|
|
filtered = all_events
|
|
if event_type is not None:
|
|
filtered = [e for e in filtered if e.type == event_type]
|
|
if since_seq is not None:
|
|
filtered = [e for e in filtered if e.seq > since_seq]
|
|
events = [
|
|
{
|
|
"seq": e.seq,
|
|
"ts": e.ts,
|
|
"type": e.type,
|
|
**e.data,
|
|
}
|
|
for e in filtered[-limit:]
|
|
]
|
|
|
|
by_type: dict[str, int] = {}
|
|
for e in filtered:
|
|
by_type[e.type] = by_type.get(e.type, 0) + 1
|
|
|
|
return {
|
|
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime()),
|
|
"capacity": self._buffer.maxlen,
|
|
"count": len(filtered),
|
|
"dropped": self._dropped,
|
|
"first_seq": filtered[0].seq if filtered else None,
|
|
"last_seq": filtered[-1].seq if filtered else None,
|
|
"summary": {"by_type": by_type},
|
|
"events": events,
|
|
}
|
|
|
|
@property
|
|
def count(self) -> int:
|
|
return len(self._buffer)
|
|
|
|
|
|
_stability_buffer = StabilityRingBuffer()
|
|
|
|
_pending_stability_reason: str | None = None
|
|
_pending_stability_error: Exception | None = None
|
|
|
|
|
|
def _write_stability_at_exit() -> None:
|
|
global _pending_stability_reason, _pending_stability_error
|
|
if _pending_stability_reason:
|
|
try:
|
|
write_stability_bundle_for_failure(_pending_stability_reason, _pending_stability_error)
|
|
except Exception:
|
|
pass
|
|
_pending_stability_reason = None
|
|
_pending_stability_error = None
|
|
|
|
|
|
def record_stability_event(event_type: str, **data: Any) -> None:
|
|
_stability_buffer.record(event_type, **data)
|
|
|
|
|
|
def get_stability_snapshot(
|
|
limit: int = DEFAULT_QUERY_LIMIT,
|
|
event_type: str | None = None,
|
|
since_seq: int | None = None,
|
|
) -> dict:
|
|
return _stability_buffer.snapshot(limit, event_type=event_type, since_seq=since_seq)
|
|
|
|
|
|
def _normalize_reason(reason: str) -> str:
|
|
return reason if SAFE_REASON_CODE.match(reason) else "unknown"
|
|
|
|
|
|
def _redact_sensitive_text(text: str) -> str:
|
|
text = sanitize_text(text)
|
|
text = _PEM_PRIVATE_KEY_PATTERN.sub("[REDACTED]", text)
|
|
return re.sub(r"\s+", " ", text).strip()
|
|
|
|
|
|
def _extract_error_name(error: Exception) -> str | None:
|
|
name = getattr(error, "name", None)
|
|
if isinstance(name, str) and SAFE_REASON_CODE.match(name):
|
|
return name
|
|
cls_name = type(error).__name__
|
|
return cls_name if SAFE_REASON_CODE.match(cls_name) else None
|
|
|
|
|
|
def _extract_error_code(error: Exception) -> str | None:
|
|
code = getattr(error, "code", None)
|
|
if isinstance(code, str) and SAFE_REASON_CODE.match(code):
|
|
return code
|
|
if isinstance(code, int) and code >= 0:
|
|
return str(code)
|
|
errno = getattr(error, "errno", None)
|
|
if isinstance(errno, int) and errno >= 0:
|
|
return str(errno)
|
|
return None
|
|
|
|
|
|
def _extract_safe_error_message(error: Exception) -> str | None:
|
|
message = getattr(error, "message", None) or str(error)
|
|
if not message:
|
|
return None
|
|
sanitized = _redact_sensitive_text(message)
|
|
if not sanitized:
|
|
return None
|
|
if len(sanitized) > MAX_SAFE_ERROR_MESSAGE_LENGTH:
|
|
sanitized = sanitized[:MAX_SAFE_ERROR_MESSAGE_LENGTH] + "..."
|
|
return sanitized
|
|
|
|
|
|
def _build_error_metadata(error: Exception) -> dict[str, str] | None:
|
|
name = _extract_error_name(error)
|
|
code = _extract_error_code(error)
|
|
message = _extract_safe_error_message(error)
|
|
if not name and not code and not message:
|
|
return None
|
|
result: dict[str, str] = {}
|
|
if name:
|
|
result["name"] = name
|
|
if code:
|
|
result["code"] = code
|
|
if message:
|
|
result["message"] = message
|
|
return result
|
|
|
|
|
|
def _resolve_bundle_dir() -> Path:
|
|
return Path(os.environ.get("YUXI_STABILITY_DIR", "data/stability"))
|
|
|
|
|
|
def _get_hostname_redacted() -> str:
|
|
return REDACTED_HOSTNAME
|
|
|
|
|
|
def _get_process_uptime_ms() -> int:
|
|
try:
|
|
return int((time.time() - __import__("psutil").Process(os.getpid()).create_time()) * 1000)
|
|
except Exception:
|
|
return 0
|
|
|
|
|
|
def _build_bundle_filename(timestamp: str, pid: int, reason: str) -> str:
|
|
normalized = _normalize_reason(reason)
|
|
return f"yuxi-stability-{timestamp}-{pid}-{normalized}.json"
|
|
|
|
|
|
def _truncate_events_to_fit(bundle: dict, snapshot: dict, max_bytes: int) -> dict:
|
|
events = snapshot["events"]
|
|
if not events:
|
|
return bundle
|
|
|
|
lo, hi = 0, len(events)
|
|
while lo < hi:
|
|
mid = (lo + hi) // 2
|
|
test_snapshot = {**snapshot, "events": events[mid:]}
|
|
test_bundle = {**bundle, "snapshot": test_snapshot}
|
|
if len(json.dumps(test_bundle, ensure_ascii=False, indent=2).encode()) <= max_bytes:
|
|
hi = mid
|
|
else:
|
|
lo = mid + 1
|
|
|
|
snapshot["events"] = events[lo:]
|
|
bundle["snapshot"] = snapshot
|
|
return bundle
|
|
|
|
|
|
def write_stability_bundle(
|
|
reason: str,
|
|
error: Exception | None = None,
|
|
include_empty: bool = False,
|
|
limit: int = DEFAULT_RING_CAPACITY,
|
|
) -> str | None:
|
|
import platform
|
|
|
|
snapshot = _stability_buffer.snapshot(limit)
|
|
if snapshot["count"] == 0 and not error and not include_empty:
|
|
return None
|
|
|
|
bundle_dir = _resolve_bundle_dir()
|
|
bundle_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
normalized_reason = _normalize_reason(reason)
|
|
error_meta = _build_error_metadata(error) if error else None
|
|
|
|
bundle = {
|
|
"version": 1,
|
|
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
"reason": normalized_reason,
|
|
"process": {
|
|
"pid": os.getpid(),
|
|
"platform": sys.platform,
|
|
"arch": platform.machine(),
|
|
"python": sys.version,
|
|
"uptime_ms": _get_process_uptime_ms(),
|
|
},
|
|
"host": {
|
|
"hostname": _get_hostname_redacted(),
|
|
},
|
|
"error": error_meta,
|
|
"snapshot": snapshot,
|
|
}
|
|
|
|
timestamp = time.strftime("%Y%m%dT%H%M%S", time.localtime())
|
|
filename = _build_bundle_filename(timestamp, os.getpid(), normalized_reason)
|
|
filepath = bundle_dir / filename
|
|
|
|
content = json.dumps(bundle, ensure_ascii=False, indent=2)
|
|
content_bytes = content.encode()
|
|
if len(content_bytes) > MAX_BUNDLE_BYTES:
|
|
logger.error("Stability bundle too large (%d bytes), truncating", len(content_bytes))
|
|
bundle = _truncate_events_to_fit(bundle, snapshot, MAX_BUNDLE_BYTES)
|
|
content = json.dumps(bundle, ensure_ascii=False, indent=2)
|
|
|
|
with open(filepath, "w", encoding="utf-8") as f:
|
|
f.write(content)
|
|
|
|
_restrict_file_permissions(filepath)
|
|
|
|
_prune_old_bundles(bundle_dir)
|
|
logger.warning("Stability bundle written: %s", filepath)
|
|
return str(filepath)
|
|
|
|
|
|
def write_stability_bundle_for_failure(reason: str, error: Exception | None = None) -> str | None:
|
|
return write_stability_bundle(reason, error=error, include_empty=True)
|
|
|
|
|
|
def _restrict_file_permissions(filepath: Path) -> None:
|
|
try:
|
|
os.chmod(filepath, 0o600)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _prune_old_bundles(bundle_dir: Path) -> None:
|
|
try:
|
|
bundles = sorted(
|
|
bundle_dir.glob("yuxi-stability-*.json"),
|
|
key=lambda p: p.stat().st_mtime,
|
|
reverse=True,
|
|
)
|
|
for p in bundles[DEFAULT_BUNDLE_RETENTION:]:
|
|
try:
|
|
p.unlink()
|
|
except OSError:
|
|
pass
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def list_stability_bundles(bundle_dir: Path | None = None) -> list[dict]:
|
|
_dir = bundle_dir or _resolve_bundle_dir()
|
|
try:
|
|
bundles = sorted(
|
|
_dir.glob("yuxi-stability-*.json"),
|
|
key=lambda p: p.stat().st_mtime,
|
|
reverse=True,
|
|
)
|
|
return [{"path": str(p), "mtime": p.stat().st_mtime} for p in bundles]
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def read_stability_bundle_file(filepath: str | Path) -> dict | None:
|
|
filepath = Path(filepath)
|
|
try:
|
|
stat = filepath.stat()
|
|
if stat.st_size > MAX_BUNDLE_BYTES:
|
|
logger.error("Stability bundle too large: %s (%d bytes)", filepath, stat.st_size)
|
|
return None
|
|
|
|
with open(filepath, encoding="utf-8") as f:
|
|
raw = f.read()
|
|
|
|
bundle = json.loads(raw)
|
|
if bundle.get("version") != 1:
|
|
logger.error("Unsupported stability bundle version: %s", bundle.get("version"))
|
|
return None
|
|
|
|
reason = bundle.get("reason", "")
|
|
if not SAFE_REASON_CODE.match(reason):
|
|
bundle["reason"] = "unknown"
|
|
|
|
host = bundle.get("host", {})
|
|
if isinstance(host, dict):
|
|
bundle["host"]["hostname"] = REDACTED_HOSTNAME
|
|
|
|
return bundle
|
|
except (json.JSONDecodeError, OSError) as e:
|
|
logger.error("Failed to read stability bundle %s: %s", filepath, e)
|
|
return None
|
|
|
|
|
|
def read_latest_stability_bundle(bundle_dir: Path | None = None) -> dict | None:
|
|
bundles = list_stability_bundles(bundle_dir)
|
|
if not bundles:
|
|
return None
|
|
return read_stability_bundle_file(bundles[0]["path"])
|
|
|
|
|
|
_original_excepthook = sys.excepthook
|
|
|
|
|
|
def install_stability_crash_hook() -> None:
|
|
import atexit
|
|
|
|
atexit.register(_write_stability_at_exit)
|
|
|
|
def _handler(signum, frame):
|
|
global _pending_stability_reason
|
|
_pending_stability_reason = f"signal-{signum}"
|
|
signal.signal(signum, signal.SIG_DFL)
|
|
os.kill(os.getpid(), signum)
|
|
|
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
try:
|
|
signal.signal(sig, _handler)
|
|
except Exception:
|
|
pass
|
|
|
|
def _excepthook(exc_type, exc_value, exc_tb):
|
|
global _pending_stability_reason, _pending_stability_error
|
|
_pending_stability_reason = "unhandled-exception"
|
|
_pending_stability_error = exc_value
|
|
_original_excepthook(exc_type, exc_value, exc_tb)
|
|
|
|
sys.excepthook = _excepthook
|