import asyncio import json import logging import os import time from pathlib import Path from yuxi.channel.cron.alert import evaluate_alerts from yuxi.channel.config.defaults import TIMEOUT from yuxi.channel.cron.delivery import DeliveryConfig, deliver_result from yuxi.channel.cron.handler_registry import resolve_handler from yuxi.channel.cron.normalize import normalize_job from yuxi.channel.cron.phase_tracker import PhaseTracker from yuxi.channel.cron.schedule import ( compute_next_run_at_ms, compute_next_with_error_tracking, ) from yuxi.channel.cron.types import ( CronAgentExecutionPhase, CronDeliveryReceipt, CronDeliveryTarget, CronFailureAlert, CronJob, CronJobState, CronJobStatus, CronPhaseRecord, CronRunDiagnostic, CronRunOutcome, CronRunStatus, DeliveryHandler, ScheduleKind, ) logger = logging.getLogger(__name__) DEFAULT_MAX_CONCURRENT = 3 DEFAULT_TICK_INTERVAL = 1.0 DEFAULT_STORE_DIR = "cron" DEFAULT_STORE_FILE = "jobs.json" DEFAULT_STATE_FILE = "jobs-state.json" DEFAULT_RUN_LOG_MAX_BYTES = 2_000_000 DEFAULT_RUN_LOG_KEEP_LINES = 2_000 DEFAULT_DIAGNOSTIC_KEEP = 200 DEFAULT_DIAGNOSTIC_LOG_MAX_BYTES = 5_000_000 DEFAULT_DIAGNOSTIC_LOG_KEEP_LINES = 5000 TOKEN_USAGE_KEYS = {"prompt_tokens", "completion_tokens", "total_tokens"} TOKEN_USAGE_NESTED_KEYS = {"usage", "token_usage", "tokens"} def _cron_dir() -> Path: from yuxi.channel.config.discovery import resolve_config_dir return Path(resolve_config_dir()) / DEFAULT_STORE_DIR def _store_path() -> Path: return _cron_dir() / DEFAULT_STORE_FILE def _state_path() -> Path: return _cron_dir() / DEFAULT_STATE_FILE def _diagnostic_log_path() -> Path: return _cron_dir() / "diagnostics.jsonl" def _run_log_path(job_id: str) -> Path: runs_dir = _cron_dir() / "runs" safe_id = job_id.strip().replace("\\", "_").replace("/", "_") return runs_dir / f"{safe_id}.jsonl" class CronEngine: def __init__( self, max_concurrent: int = DEFAULT_MAX_CONCURRENT, tick_interval: float = DEFAULT_TICK_INTERVAL, store_path: str | None = None, state_path: str | None = None, ): self._jobs: dict[str, CronJob] = {} self._semaphore = asyncio.Semaphore(max_concurrent) self._tick_interval = tick_interval self._stop_event = asyncio.Event() self._task: asyncio.Task | None = None self._store_path = Path(store_path) if store_path else _store_path() self._state_path = Path(state_path) if state_path else _state_path() self._cron_dir = self._store_path.parent self._loaded = False self._delivery_handler: DeliveryHandler | None = None self._delivery_config = DeliveryConfig() self._diagnostics: list[CronRunDiagnostic] = [] self._running_tasks: set[asyncio.Task] = set() def _cron_dir_path(self) -> Path: return self._cron_dir def _diagnostic_log_path(self) -> Path: return self._cron_dir / "diagnostics.jsonl" def _run_log_path(self, job_id: str) -> Path: runs_dir = self._cron_dir / "runs" safe_id = job_id.strip().replace("\\", "_").replace("/", "_") return runs_dir / f"{safe_id}.jsonl" @property def is_running(self) -> bool: return self._task is not None and not self._task.done() def set_delivery_handler(self, handler: DeliveryHandler) -> None: self._delivery_handler = handler def set_delivery_config(self, config: DeliveryConfig) -> None: self._delivery_config = config def _ensure_loaded(self): if not self._loaded: self._load_store() def _load_store(self): self._loaded = True try: self._store_path.parent.mkdir(parents=True, exist_ok=True) except OSError: pass self._load_diagnostics() state_data: dict[str, dict] = {} if self._state_path.exists(): try: raw = self._state_path.read_text(encoding="utf-8") state_raw = json.loads(raw) if isinstance(state_raw, dict): state_data = state_raw.get("jobs", {}) if isinstance(state_data, list): state_data = {item.get("id", ""): item for item in state_data if isinstance(item, dict)} except (json.JSONDecodeError, OSError): logger.warning("CronEngine: failed to load state, starting fresh") if not self._store_path.exists(): return try: raw = self._store_path.read_text(encoding="utf-8") data = json.loads(raw) jobs_data = data.get("jobs", []) if isinstance(data, dict) else [] except (json.JSONDecodeError, OSError): logger.warning("CronEngine: failed to load store, starting fresh") return for jd in jobs_data: if not isinstance(jd, dict): continue try: job = self._deserialize_job(jd, state_data.get(jd.get("id", ""), {})) normalize_job(job) self._jobs[job.id] = job except Exception: logger.exception("CronEngine: failed to deserialize job '%s'", jd.get("id", "?")) logger.info("CronEngine: loaded %d jobs from %s", len(self._jobs), self._store_path) def _load_diagnostics(self): diag_path = self._diagnostic_log_path() if not diag_path.exists(): return try: self._diagnostics = [] with open(diag_path, encoding="utf-8") as f: for line in f: line = line.strip() if not line: continue try: data = json.loads(line) self._diagnostics.append( CronRunDiagnostic( run_id=data.get("run_id", ""), job_id=data.get("job_id", ""), outcome=CronRunOutcome(data.get("outcome", "success")), phases=[ CronPhaseRecord( phase=CronAgentExecutionPhase(p["phase"]), started_at_ms=p["started_at_ms"], finished_at_ms=p.get("finished_at_ms"), error=p.get("error"), ) for p in data.get("phases", []) ], total_duration_ms=data.get("total_duration_ms", 0), error=data.get("error"), token_usage=data.get("token_usage", {}), delivery_receipts=[ CronDeliveryReceipt( target=CronDeliveryTarget.from_dict(r["target"]), success=r.get("success", False), message_id=r.get("message_id"), error=r.get("error"), retries=r.get("retries", 0), sent_at_ms=r.get("sent_at_ms", 0), duration_ms=r.get("duration_ms", 0), ) for r in data.get("delivery_receipts", []) ], started_at_ms=data.get("started_at_ms", 0), finished_at_ms=data.get("finished_at_ms", 0), ) ) except (json.JSONDecodeError, KeyError, TypeError): logger.warning("CronEngine: failed to parse diagnostic line") self._diagnostics = self._diagnostics[-DEFAULT_DIAGNOSTIC_KEEP:] logger.info("CronEngine: loaded %d diagnostics from %s", len(self._diagnostics), diag_path) except OSError: logger.warning("CronEngine: failed to load diagnostics, starting fresh") def _save_store(self): try: self._store_path.parent.mkdir(parents=True, exist_ok=True) data = { "version": 1, "jobs": [self._serialize_job(j) for j in self._jobs.values()], } tmp_path = self._store_path.with_suffix(".tmp") tmp_path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") os.replace(tmp_path, self._store_path) except OSError: logger.exception("CronEngine: failed to save store") def _save_state(self): try: self._state_path.parent.mkdir(parents=True, exist_ok=True) state_data = { "version": 1, "jobs": {j.id: j.state.to_dict() for j in self._jobs.values()}, } tmp_path = self._state_path.with_suffix(".tmp") tmp_path.write_text(json.dumps(state_data, ensure_ascii=False, indent=2), encoding="utf-8") os.replace(tmp_path, self._state_path) except OSError: logger.exception("CronEngine: failed to save state") def _save_all(self): self._save_store() self._save_state() def _serialize_job(self, job: CronJob) -> dict: d = job.to_dict() d.pop("handler", None) d.pop("handler_args", None) return d def _deserialize_job(self, jd: dict, state_data: dict | None = None) -> CronJob: state_dict = jd.get("state", {}) if state_data: state_dict = {**state_dict, **state_data} job = CronJob( id=jd["id"], name=jd.get("name", ""), handler=resolve_handler(jd.get("handler_name", "")), handler_name=jd.get("handler_name", ""), handler_args={}, schedule_kind=ScheduleKind(jd.get("schedule_kind", "cron")), schedule_value=jd.get("schedule_value", jd.get("cron_expr", "* * * * *")), schedule_tz=jd.get("schedule_tz"), stagger_ms=jd.get("stagger_ms", 0), every_ms=jd.get("every_ms"), anchor_ms=jd.get("anchor_ms"), status=CronJobStatus(jd.get("status", "active")), enabled=jd.get("enabled", True), max_runs=jd.get("max_runs", 0), delete_after_run=jd.get("delete_after_run", False), error_threshold=jd.get("error_threshold", 5), delivery=[CronDeliveryTarget.from_dict(t) for t in jd.get("delivery", []) if isinstance(t, dict)], failure_alert=CronFailureAlert.from_dict(jd.get("failure_alert")), state=CronJobState.from_dict(state_dict), created_at=jd.get("created_at", time.time()), updated_at=jd.get("updated_at", time.time()), ) return job def add_job(self, job: CronJob) -> None: self._ensure_loaded() if job.id in self._jobs: raise ValueError(f"Job '{job.id}' already registered") normalize_job(job) job.state.next_run_at_ms = compute_next_run_at_ms(job) self._jobs[job.id] = job self._save_all() def remove_job(self, job_id: str) -> None: self._ensure_loaded() self._jobs.pop(job_id, None) self._save_all() def get_job(self, job_id: str) -> CronJob | None: self._ensure_loaded() return self._jobs.get(job_id) def list_jobs(self) -> list[CronJob]: self._ensure_loaded() return list(self._jobs.values()) def update_job(self, job_id: str, **kwargs) -> CronJob | None: self._ensure_loaded() job = self._jobs.get(job_id) if job is None: return None allowed = frozenset({ "name", "enabled", "schedule_kind", "schedule_value", "schedule_tz", "stagger_ms", "every_ms", "anchor_ms", "max_runs", "error_threshold", "delivery", "failure_alert", "delete_after_run", }) for k, v in kwargs.items(): if k in allowed and hasattr(job, k): setattr(job, k, v) job.updated_at = time.time() normalize_job(job) self._save_all() return job def pause_job(self, job_id: str) -> None: self._ensure_loaded() job = self._jobs.get(job_id) if job: job.status = CronJobStatus.PAUSED job.updated_at = time.time() self._save_all() def resume_job(self, job_id: str) -> None: self._ensure_loaded() job = self._jobs.get(job_id) if job: job.status = CronJobStatus.ACTIVE job.state.next_run_at_ms = compute_next_run_at_ms(job) job.updated_at = time.time() self._save_all() async def start(self) -> None: self._ensure_loaded() if self.is_running: return self._stop_event.clear() self._task = asyncio.create_task(self._loop(), name="cron-engine") logger.info("CronEngine started with %d jobs", len(self._jobs)) async def stop(self) -> None: if not self.is_running: return self._stop_event.set() if self._task and not self._task.done(): self._task.cancel() running = [t for t in self._running_tasks if not t.done()] if running: for task in running: task.cancel() try: await asyncio.wait_for(asyncio.gather(*running, return_exceptions=True), timeout=TIMEOUT.cron.gather) except TimeoutError: logger.warning("CronEngine: some running jobs did not cancel within 15s") logger.info("CronEngine cancelled %d running jobs", len(running)) if self._task: try: await asyncio.wait_for(self._task, timeout=TIMEOUT.cron.task) except (asyncio.CancelledError, TimeoutError): pass finally: if not self._task.done(): logger.warning("CronEngine: main task did not terminate gracefully") self._task = None self._running_tasks.clear() self._save_all() logger.info("CronEngine stopped") async def force_run(self, job_id: str) -> dict: self._ensure_loaded() job = self._jobs.get(job_id) if job is None: return {"success": False, "error": f"Job '{job_id}' not found"} if job.state.running_at_ms is not None: return {"success": False, "error": f"Job '{job_id}' is already running"} return await self._run_job(job) async def _loop(self) -> None: while not self._stop_event.is_set(): try: await self._tick() except Exception: logger.exception("CronEngine tick error") try: await asyncio.wait_for( self._stop_event.wait(), timeout=self._tick_interval, ) break except TimeoutError: pass self._save_all() async def _tick(self) -> None: now_ms = int(time.time() * 1000) due_jobs = [ j for j in self._jobs.values() if j.status == CronJobStatus.ACTIVE and j.enabled and j.state.next_run_at_ms is not None and now_ms >= j.state.next_run_at_ms and j.state.running_at_ms is None ] if not due_jobs: return tasks = [asyncio.create_task(self._run_job(j), name=f"cron-job-{j.id}") for j in due_jobs] self._running_tasks.update(tasks) try: await asyncio.wait_for( asyncio.gather(*tasks, return_exceptions=True), timeout=TIMEOUT.cron.delivery, ) except TimeoutError: logger.warning("CronEngine tick timeout: %d jobs did not complete in 300s", len(due_jobs)) for task in tasks: if not task.done(): task.cancel() await asyncio.gather(*tasks, return_exceptions=True) finally: for task in tasks: if not task.done(): task.cancel() try: await task except asyncio.CancelledError: pass self._running_tasks.difference_update(tasks) async def _run_job(self, job: CronJob) -> dict: async with self._semaphore: if job.state.running_at_ms is not None: return {"success": False, "error": f"Job '{job.id}' is already running"} if job.max_runs > 0 and job.state.run_count >= job.max_runs: job.status = CronJobStatus.EXPIRED self._save_all() return {"success": False, "error": "max_runs reached"} tracker = PhaseTracker(job) diagnostic = tracker.diagnostic exec_phase = tracker.start_executing() run_start_mono = time.monotonic() job.state.running_at_ms = diagnostic.started_at_ms try: result = await job.handler(**(job.handler_args or {})) tracker.finish_executing(exec_phase) error = None outcome = CronRunOutcome.SUCCESS job.state.last_run_status = CronRunStatus.OK job.state.last_error = None job.state.run_count += 1 job.state.consecutive_errors = 0 job.state.consecutive_skipped = 0 job.state.next_run_at_ms = compute_next_with_error_tracking(job) job.status = CronJobStatus.ACTIVE summ_phase = tracker.start_summarizing() tracker.finish_summarizing(summ_phase) deliver_phase = tracker.start_delivering() delivery_receipts: list = [] if job.delivery and self._delivery_handler: delivery_receipts = await deliver_result( self._delivery_handler, job, result if isinstance(result, dict) else {"result": str(result)}, None, diagnostic, self._delivery_config, ) tracker.finish_delivering(deliver_phase) receipt_dicts = [r.to_dict() for r in delivery_receipts] except Exception as e: try: tracker.finish_executing(exec_phase, str(e)) except Exception: logger.exception("CronEngine: failed to finish executing tracker for job '%s'", job.id) error = str(e) outcome = CronRunOutcome.FAILED job.state.error_count += 1 job.state.consecutive_errors += 1 job.state.last_run_status = CronRunStatus.ERROR job.state.last_error = error job.state.next_run_at_ms = compute_next_with_error_tracking(job) if job.state.consecutive_errors >= job.error_threshold: job.status = CronJobStatus.FAILED deliver_phase = tracker.start_delivering() delivery_receipts: list = [] if job.delivery and self._delivery_handler: delivery_receipts = await deliver_result( self._delivery_handler, job, None, error, diagnostic, self._delivery_config, ) tracker.finish_delivering(deliver_phase) receipt_dicts = [r.to_dict() for r in delivery_receipts] result = None logger.exception( "CronEngine job '%s' failed (consecutive: %d)", job.id, job.state.consecutive_errors, ) duration_ms = int((time.monotonic() - run_start_mono) * 1000) token_usage = _extract_token_usage(result) if not error else {} diagnostic = tracker.close(outcome, error, token_usage) self._append_diagnostic(diagnostic) job.state.last_run_at_ms = diagnostic.started_at_ms job.state.last_duration_ms = duration_ms job.state.running_at_ms = None job.updated_at = time.time() self._append_run_log( job, { "ts": diagnostic.started_at_ms, "job_id": job.id, "action": "finished", "status": "ok" if outcome == CronRunOutcome.SUCCESS else "error", "run_id": diagnostic.run_id, "run_at_ms": diagnostic.started_at_ms, "duration_ms": duration_ms, "next_run_at_ms": job.state.next_run_at_ms, "consecutive_errors": job.state.consecutive_errors, **({"error": error} if error else {}), **( {"result": result if isinstance(result, dict) else {"result": str(result)}} if result is not None else {} ), "phases": [p.to_dict() for p in diagnostic.phases], "delivery": receipt_dicts, }, ) if job.delete_after_run and job.schedule_kind == ScheduleKind.AT: self._jobs.pop(job.id, None) logger.info("CronEngine: deleted one-shot job '%s'", job.id) await evaluate_alerts(self._delivery_handler, job) self._save_all() if outcome == CronRunOutcome.SUCCESS: return { "success": True, "job_id": job.id, "run_id": diagnostic.run_id, "diagnostic": diagnostic.to_dict(), } return { "success": False, "error": error, "run_id": diagnostic.run_id, "diagnostic": diagnostic.to_dict(), } def _append_run_log(self, job: CronJob, entry: dict): try: log_path = self._run_log_path(job.id) log_path.parent.mkdir(parents=True, exist_ok=True) line = json.dumps(entry, ensure_ascii=False) + "\n" with open(log_path, "a", encoding="utf-8") as f: f.write(line) self._prune_run_log(log_path) except OSError: logger.exception("CronEngine: failed to write run log for '%s'", job.id) def _prune_run_log(self, log_path: Path): stat = log_path.stat() if stat.st_size <= DEFAULT_RUN_LOG_MAX_BYTES: return try: lines = log_path.read_text(encoding="utf-8").strip().split("\n") kept = lines[-DEFAULT_RUN_LOG_KEEP_LINES:] log_path.write_text("\n".join(kept) + "\n", encoding="utf-8") except OSError: pass def read_run_log(self, job_id: str, limit: int = 50, offset: int = 0) -> dict: log_path = self._run_log_path(job_id) if not log_path.exists(): return {"entries": [], "total": 0, "offset": offset, "limit": limit, "has_more": False} try: raw = log_path.read_text(encoding="utf-8") lines = [json.loads(line) for line in raw.strip().split("\n") if line.strip()] except (OSError, json.JSONDecodeError): return {"entries": [], "total": 0, "offset": offset, "limit": limit, "has_more": False} total = len(lines) start = max(0, min(total, offset)) end = min(total, start + limit) entries = lines[start:end] return { "entries": entries, "total": total, "offset": offset, "limit": limit, "has_more": end < total, } def _append_diagnostic(self, diagnostic: CronRunDiagnostic) -> None: self._diagnostics.append(diagnostic) if len(self._diagnostics) > DEFAULT_DIAGNOSTIC_KEEP: self._diagnostics = self._diagnostics[-DEFAULT_DIAGNOSTIC_KEEP:] self._write_diagnostic_log(diagnostic) def _write_diagnostic_log(self, diagnostic: CronRunDiagnostic) -> None: try: diag_path = self._diagnostic_log_path() diag_path.parent.mkdir(parents=True, exist_ok=True) line = json.dumps(diagnostic.to_dict(), ensure_ascii=False) + "\n" with open(diag_path, "a", encoding="utf-8") as f: f.write(line) self._prune_diagnostic_log(diag_path) except OSError: logger.exception("CronEngine: failed to write diagnostic log") def _prune_diagnostic_log(self, diag_path: Path) -> None: try: stat = diag_path.stat() except OSError: return if stat.st_size <= DEFAULT_DIAGNOSTIC_LOG_MAX_BYTES: return try: lines = diag_path.read_text(encoding="utf-8").strip().split("\n") kept = lines[-DEFAULT_DIAGNOSTIC_LOG_KEEP_LINES:] diag_path.write_text("\n".join(kept) + "\n", encoding="utf-8") except OSError: pass def read_diagnostics(self, job_id: str | None = None, limit: int = 50, offset: int = 0) -> dict: all_diags = self._read_all_diagnostics() filtered = all_diags if job_id: filtered = [d for d in filtered if d.job_id == job_id] total = len(filtered) start = max(0, min(total, offset)) end = min(total, start + limit) entries = [d.to_dict() for d in filtered[start:end]] return { "entries": entries, "total": total, "offset": offset, "limit": limit, "has_more": end < total, } def _read_all_diagnostics(self) -> list[CronRunDiagnostic]: diag_path = self._diagnostic_log_path() if not diag_path.exists(): return list(self._diagnostics) memory_ids = {d.run_id for d in self._diagnostics if d.run_id} result = list(self._diagnostics) try: raw = diag_path.read_text(encoding="utf-8") for line in raw.strip().split("\n"): if not line.strip(): continue try: data = json.loads(line) if data.get("run_id") in memory_ids: continue result.append( CronRunDiagnostic( run_id=data.get("run_id", ""), job_id=data.get("job_id", ""), outcome=CronRunOutcome(data.get("outcome", "success")), phases=[ CronPhaseRecord( phase=CronAgentExecutionPhase(p["phase"]), started_at_ms=p["started_at_ms"], finished_at_ms=p.get("finished_at_ms"), error=p.get("error"), ) for p in data.get("phases", []) ], total_duration_ms=data.get("total_duration_ms", 0), error=data.get("error"), token_usage=data.get("token_usage", {}), delivery_receipts=[ CronDeliveryReceipt( target=CronDeliveryTarget.from_dict(r["target"]), success=r.get("success", False), message_id=r.get("message_id"), error=r.get("error"), retries=r.get("retries", 0), sent_at_ms=r.get("sent_at_ms", 0), duration_ms=r.get("duration_ms", 0), ) for r in data.get("delivery_receipts", []) ], started_at_ms=data.get("started_at_ms", 0), finished_at_ms=data.get("finished_at_ms", 0), ) ) except (json.JSONDecodeError, KeyError, TypeError): pass except OSError: pass return sorted(result, key=lambda d: d.started_at_ms) def list_diagnostics_for_job(self, job_id: str, limit: int = 10) -> list[dict]: all_diags = self._read_all_diagnostics() matching = [d for d in all_diags if d.job_id == job_id] return [d.to_dict() for d in matching[-limit:]] def _extract_token_usage(result: dict | None) -> dict: if not result or not isinstance(result, dict): return {} usage = {} for key in TOKEN_USAGE_KEYS: if key in result and isinstance(result[key], (int, float)): usage[key] = result[key] for nested_key in TOKEN_USAGE_NESTED_KEYS: if nested_key in result and isinstance(result[nested_key], dict): for k, v in result[nested_key].items(): if isinstance(v, (int, float)): usage[k] = v return usage _engine: CronEngine | None = None def get_cron_engine() -> CronEngine: global _engine if _engine is None: _engine = CronEngine() return _engine