import json import logging import os import re import signal import sys import time from collections import deque from dataclasses import dataclass, field from pathlib import Path from typing import Any from yuxi.channel.security.log_sanitizer import sanitize_text logger = logging.getLogger(__name__) DEFAULT_RING_CAPACITY = 1000 DEFAULT_QUERY_LIMIT = 50 MAX_BUNDLE_BYTES = 5 * 1024 * 1024 DEFAULT_BUNDLE_RETENTION = 20 MAX_SAFE_ERROR_MESSAGE_LENGTH = 500 SAFE_REASON_CODE = re.compile(r"^[A-Za-z0-9_.:-]{1,120}$") REDACTED_HOSTNAME = "" _PEM_PRIVATE_KEY_PATTERN = re.compile( r"-----BEGIN\s+(?:RSA\s+)?PRIVATE\s+KEY-----[\s\S]*?" r"-----END\s+(?:RSA\s+)?PRIVATE\s+KEY-----", ) @dataclass class StabilityEvent: seq: int ts: float type: str data: dict[str, Any] = field(default_factory=dict) class StabilityRingBuffer: def __init__(self, capacity: int = DEFAULT_RING_CAPACITY): self._buffer: deque[StabilityEvent] = deque(maxlen=capacity) self._seq = 0 self._dropped = 0 def record(self, event_type: str, **data: Any) -> None: self._seq += 1 event = StabilityEvent( seq=self._seq, ts=time.time(), type=event_type, data=data, ) if len(self._buffer) >= self._buffer.maxlen: self._dropped += 1 self._buffer.append(event) def query( self, limit: int = DEFAULT_QUERY_LIMIT, event_type: str | None = None, since_seq: int | None = None, ) -> list[dict]: events = list(self._buffer) if event_type is not None: events = [e for e in events if e.type == event_type] if since_seq is not None: events = [e for e in events if e.seq > since_seq] events = events[-limit:] return [ { "seq": e.seq, "ts": e.ts, "type": e.type, **e.data, } for e in events ] def snapshot( self, limit: int = DEFAULT_QUERY_LIMIT, event_type: str | None = None, since_seq: int | None = None, ) -> dict: all_events = list(self._buffer) filtered = all_events if event_type is not None: filtered = [e for e in filtered if e.type == event_type] if since_seq is not None: filtered = [e for e in filtered if e.seq > since_seq] events = [ { "seq": e.seq, "ts": e.ts, "type": e.type, **e.data, } for e in filtered[-limit:] ] by_type: dict[str, int] = {} for e in filtered: by_type[e.type] = by_type.get(e.type, 0) + 1 return { "generated_at": time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime()), "capacity": self._buffer.maxlen, "count": len(filtered), "dropped": self._dropped, "first_seq": filtered[0].seq if filtered else None, "last_seq": filtered[-1].seq if filtered else None, "summary": {"by_type": by_type}, "events": events, } @property def count(self) -> int: return len(self._buffer) _stability_buffer = StabilityRingBuffer() _pending_stability_reason: str | None = None _pending_stability_error: Exception | None = None def _write_stability_at_exit() -> None: global _pending_stability_reason, _pending_stability_error if _pending_stability_reason: try: write_stability_bundle_for_failure(_pending_stability_reason, _pending_stability_error) except Exception: pass _pending_stability_reason = None _pending_stability_error = None def record_stability_event(event_type: str, **data: Any) -> None: _stability_buffer.record(event_type, **data) def get_stability_snapshot( limit: int = DEFAULT_QUERY_LIMIT, event_type: str | None = None, since_seq: int | None = None, ) -> dict: return _stability_buffer.snapshot(limit, event_type=event_type, since_seq=since_seq) def _normalize_reason(reason: str) -> str: return reason if SAFE_REASON_CODE.match(reason) else "unknown" def _redact_sensitive_text(text: str) -> str: text = sanitize_text(text) text = _PEM_PRIVATE_KEY_PATTERN.sub("[REDACTED]", text) return re.sub(r"\s+", " ", text).strip() def _extract_error_name(error: Exception) -> str | None: name = getattr(error, "name", None) if isinstance(name, str) and SAFE_REASON_CODE.match(name): return name cls_name = type(error).__name__ return cls_name if SAFE_REASON_CODE.match(cls_name) else None def _extract_error_code(error: Exception) -> str | None: code = getattr(error, "code", None) if isinstance(code, str) and SAFE_REASON_CODE.match(code): return code if isinstance(code, int) and code >= 0: return str(code) errno = getattr(error, "errno", None) if isinstance(errno, int) and errno >= 0: return str(errno) return None def _extract_safe_error_message(error: Exception) -> str | None: message = getattr(error, "message", None) or str(error) if not message: return None sanitized = _redact_sensitive_text(message) if not sanitized: return None if len(sanitized) > MAX_SAFE_ERROR_MESSAGE_LENGTH: sanitized = sanitized[:MAX_SAFE_ERROR_MESSAGE_LENGTH] + "..." return sanitized def _build_error_metadata(error: Exception) -> dict[str, str] | None: name = _extract_error_name(error) code = _extract_error_code(error) message = _extract_safe_error_message(error) if not name and not code and not message: return None result: dict[str, str] = {} if name: result["name"] = name if code: result["code"] = code if message: result["message"] = message return result def _resolve_bundle_dir() -> Path: return Path(os.environ.get("YUXI_STABILITY_DIR", "data/stability")) def _get_hostname_redacted() -> str: return REDACTED_HOSTNAME def _get_process_uptime_ms() -> int: try: return int((time.time() - __import__("psutil").Process(os.getpid()).create_time()) * 1000) except Exception: return 0 def _build_bundle_filename(timestamp: str, pid: int, reason: str) -> str: normalized = _normalize_reason(reason) return f"yuxi-stability-{timestamp}-{pid}-{normalized}.json" def _truncate_events_to_fit(bundle: dict, snapshot: dict, max_bytes: int) -> dict: events = snapshot["events"] if not events: return bundle lo, hi = 0, len(events) while lo < hi: mid = (lo + hi) // 2 test_snapshot = {**snapshot, "events": events[mid:]} test_bundle = {**bundle, "snapshot": test_snapshot} if len(json.dumps(test_bundle, ensure_ascii=False, indent=2).encode()) <= max_bytes: hi = mid else: lo = mid + 1 snapshot["events"] = events[lo:] bundle["snapshot"] = snapshot return bundle def write_stability_bundle( reason: str, error: Exception | None = None, include_empty: bool = False, limit: int = DEFAULT_RING_CAPACITY, ) -> str | None: import platform snapshot = _stability_buffer.snapshot(limit) if snapshot["count"] == 0 and not error and not include_empty: return None bundle_dir = _resolve_bundle_dir() bundle_dir.mkdir(parents=True, exist_ok=True) normalized_reason = _normalize_reason(reason) error_meta = _build_error_metadata(error) if error else None bundle = { "version": 1, "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "reason": normalized_reason, "process": { "pid": os.getpid(), "platform": sys.platform, "arch": platform.machine(), "python": sys.version, "uptime_ms": _get_process_uptime_ms(), }, "host": { "hostname": _get_hostname_redacted(), }, "error": error_meta, "snapshot": snapshot, } timestamp = time.strftime("%Y%m%dT%H%M%S", time.localtime()) filename = _build_bundle_filename(timestamp, os.getpid(), normalized_reason) filepath = bundle_dir / filename content = json.dumps(bundle, ensure_ascii=False, indent=2) content_bytes = content.encode() if len(content_bytes) > MAX_BUNDLE_BYTES: logger.error("Stability bundle too large (%d bytes), truncating", len(content_bytes)) bundle = _truncate_events_to_fit(bundle, snapshot, MAX_BUNDLE_BYTES) content = json.dumps(bundle, ensure_ascii=False, indent=2) with open(filepath, "w", encoding="utf-8") as f: f.write(content) _restrict_file_permissions(filepath) _prune_old_bundles(bundle_dir) logger.warning("Stability bundle written: %s", filepath) return str(filepath) def write_stability_bundle_for_failure(reason: str, error: Exception | None = None) -> str | None: return write_stability_bundle(reason, error=error, include_empty=True) def _restrict_file_permissions(filepath: Path) -> None: try: os.chmod(filepath, 0o600) except Exception: pass def _prune_old_bundles(bundle_dir: Path) -> None: try: bundles = sorted( bundle_dir.glob("yuxi-stability-*.json"), key=lambda p: p.stat().st_mtime, reverse=True, ) for p in bundles[DEFAULT_BUNDLE_RETENTION:]: try: p.unlink() except OSError: pass except Exception: pass def list_stability_bundles(bundle_dir: Path | None = None) -> list[dict]: _dir = bundle_dir or _resolve_bundle_dir() try: bundles = sorted( _dir.glob("yuxi-stability-*.json"), key=lambda p: p.stat().st_mtime, reverse=True, ) return [{"path": str(p), "mtime": p.stat().st_mtime} for p in bundles] except Exception: return [] def read_stability_bundle_file(filepath: str | Path) -> dict | None: filepath = Path(filepath) try: stat = filepath.stat() if stat.st_size > MAX_BUNDLE_BYTES: logger.error("Stability bundle too large: %s (%d bytes)", filepath, stat.st_size) return None with open(filepath, encoding="utf-8") as f: raw = f.read() bundle = json.loads(raw) if bundle.get("version") != 1: logger.error("Unsupported stability bundle version: %s", bundle.get("version")) return None reason = bundle.get("reason", "") if not SAFE_REASON_CODE.match(reason): bundle["reason"] = "unknown" host = bundle.get("host", {}) if isinstance(host, dict): bundle["host"]["hostname"] = REDACTED_HOSTNAME return bundle except (json.JSONDecodeError, OSError) as e: logger.error("Failed to read stability bundle %s: %s", filepath, e) return None def read_latest_stability_bundle(bundle_dir: Path | None = None) -> dict | None: bundles = list_stability_bundles(bundle_dir) if not bundles: return None return read_stability_bundle_file(bundles[0]["path"]) _original_excepthook = sys.excepthook def install_stability_crash_hook() -> None: import atexit atexit.register(_write_stability_at_exit) def _handler(signum, frame): global _pending_stability_reason _pending_stability_reason = f"signal-{signum}" signal.signal(signum, signal.SIG_DFL) os.kill(os.getpid(), signum) for sig in (signal.SIGTERM, signal.SIGINT): try: signal.signal(sig, _handler) except Exception: pass def _excepthook(exc_type, exc_value, exc_tb): global _pending_stability_reason, _pending_stability_error _pending_stability_reason = "unhandled-exception" _pending_stability_error = exc_value _original_excepthook(exc_type, exc_value, exc_tb) sys.excepthook = _excepthook