756 lines
30 KiB
Python
756 lines
30 KiB
Python
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from yuxi.channel.cron.alert import evaluate_alerts
|
|
from yuxi.channel.config.defaults import TIMEOUT
|
|
from yuxi.channel.cron.delivery import DeliveryConfig, deliver_result
|
|
from yuxi.channel.cron.handler_registry import resolve_handler
|
|
from yuxi.channel.cron.normalize import normalize_job
|
|
from yuxi.channel.cron.phase_tracker import PhaseTracker
|
|
from yuxi.channel.cron.schedule import (
|
|
compute_next_run_at_ms,
|
|
compute_next_with_error_tracking,
|
|
)
|
|
from yuxi.channel.cron.types import (
|
|
CronAgentExecutionPhase,
|
|
CronDeliveryReceipt,
|
|
CronDeliveryTarget,
|
|
CronFailureAlert,
|
|
CronJob,
|
|
CronJobState,
|
|
CronJobStatus,
|
|
CronPhaseRecord,
|
|
CronRunDiagnostic,
|
|
CronRunOutcome,
|
|
CronRunStatus,
|
|
DeliveryHandler,
|
|
ScheduleKind,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DEFAULT_MAX_CONCURRENT = 3
|
|
DEFAULT_TICK_INTERVAL = 1.0
|
|
DEFAULT_STORE_DIR = "cron"
|
|
DEFAULT_STORE_FILE = "jobs.json"
|
|
DEFAULT_STATE_FILE = "jobs-state.json"
|
|
DEFAULT_RUN_LOG_MAX_BYTES = 2_000_000
|
|
DEFAULT_RUN_LOG_KEEP_LINES = 2_000
|
|
DEFAULT_DIAGNOSTIC_KEEP = 200
|
|
DEFAULT_DIAGNOSTIC_LOG_MAX_BYTES = 5_000_000
|
|
DEFAULT_DIAGNOSTIC_LOG_KEEP_LINES = 5000
|
|
TOKEN_USAGE_KEYS = {"prompt_tokens", "completion_tokens", "total_tokens"}
|
|
TOKEN_USAGE_NESTED_KEYS = {"usage", "token_usage", "tokens"}
|
|
|
|
|
|
def _cron_dir() -> Path:
|
|
from yuxi.channel.config.discovery import resolve_config_dir
|
|
|
|
return Path(resolve_config_dir()) / DEFAULT_STORE_DIR
|
|
|
|
|
|
def _store_path() -> Path:
|
|
return _cron_dir() / DEFAULT_STORE_FILE
|
|
|
|
|
|
def _state_path() -> Path:
|
|
return _cron_dir() / DEFAULT_STATE_FILE
|
|
|
|
|
|
def _diagnostic_log_path() -> Path:
|
|
return _cron_dir() / "diagnostics.jsonl"
|
|
|
|
|
|
def _run_log_path(job_id: str) -> Path:
|
|
runs_dir = _cron_dir() / "runs"
|
|
safe_id = job_id.strip().replace("\\", "_").replace("/", "_")
|
|
return runs_dir / f"{safe_id}.jsonl"
|
|
|
|
|
|
class CronEngine:
|
|
def __init__(
|
|
self,
|
|
max_concurrent: int = DEFAULT_MAX_CONCURRENT,
|
|
tick_interval: float = DEFAULT_TICK_INTERVAL,
|
|
store_path: str | None = None,
|
|
state_path: str | None = None,
|
|
):
|
|
self._jobs: dict[str, CronJob] = {}
|
|
self._semaphore = asyncio.Semaphore(max_concurrent)
|
|
self._tick_interval = tick_interval
|
|
self._stop_event = asyncio.Event()
|
|
self._task: asyncio.Task | None = None
|
|
self._store_path = Path(store_path) if store_path else _store_path()
|
|
self._state_path = Path(state_path) if state_path else _state_path()
|
|
self._cron_dir = self._store_path.parent
|
|
self._loaded = False
|
|
self._delivery_handler: DeliveryHandler | None = None
|
|
self._delivery_config = DeliveryConfig()
|
|
self._diagnostics: list[CronRunDiagnostic] = []
|
|
self._running_tasks: set[asyncio.Task] = set()
|
|
|
|
def _cron_dir_path(self) -> Path:
|
|
return self._cron_dir
|
|
|
|
def _diagnostic_log_path(self) -> Path:
|
|
return self._cron_dir / "diagnostics.jsonl"
|
|
|
|
def _run_log_path(self, job_id: str) -> Path:
|
|
runs_dir = self._cron_dir / "runs"
|
|
safe_id = job_id.strip().replace("\\", "_").replace("/", "_")
|
|
return runs_dir / f"{safe_id}.jsonl"
|
|
|
|
@property
|
|
def is_running(self) -> bool:
|
|
return self._task is not None and not self._task.done()
|
|
|
|
def set_delivery_handler(self, handler: DeliveryHandler) -> None:
|
|
self._delivery_handler = handler
|
|
|
|
def set_delivery_config(self, config: DeliveryConfig) -> None:
|
|
self._delivery_config = config
|
|
|
|
def _ensure_loaded(self):
|
|
if not self._loaded:
|
|
self._load_store()
|
|
|
|
def _load_store(self):
|
|
self._loaded = True
|
|
try:
|
|
self._store_path.parent.mkdir(parents=True, exist_ok=True)
|
|
except OSError:
|
|
pass
|
|
|
|
self._load_diagnostics()
|
|
|
|
state_data: dict[str, dict] = {}
|
|
if self._state_path.exists():
|
|
try:
|
|
raw = self._state_path.read_text(encoding="utf-8")
|
|
state_raw = json.loads(raw)
|
|
if isinstance(state_raw, dict):
|
|
state_data = state_raw.get("jobs", {})
|
|
if isinstance(state_data, list):
|
|
state_data = {item.get("id", ""): item for item in state_data if isinstance(item, dict)}
|
|
except (json.JSONDecodeError, OSError):
|
|
logger.warning("CronEngine: failed to load state, starting fresh")
|
|
|
|
if not self._store_path.exists():
|
|
return
|
|
try:
|
|
raw = self._store_path.read_text(encoding="utf-8")
|
|
data = json.loads(raw)
|
|
jobs_data = data.get("jobs", []) if isinstance(data, dict) else []
|
|
except (json.JSONDecodeError, OSError):
|
|
logger.warning("CronEngine: failed to load store, starting fresh")
|
|
return
|
|
|
|
for jd in jobs_data:
|
|
if not isinstance(jd, dict):
|
|
continue
|
|
try:
|
|
job = self._deserialize_job(jd, state_data.get(jd.get("id", ""), {}))
|
|
normalize_job(job)
|
|
self._jobs[job.id] = job
|
|
except Exception:
|
|
logger.exception("CronEngine: failed to deserialize job '%s'", jd.get("id", "?"))
|
|
|
|
logger.info("CronEngine: loaded %d jobs from %s", len(self._jobs), self._store_path)
|
|
|
|
def _load_diagnostics(self):
|
|
diag_path = self._diagnostic_log_path()
|
|
if not diag_path.exists():
|
|
return
|
|
try:
|
|
self._diagnostics = []
|
|
with open(diag_path, encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
data = json.loads(line)
|
|
self._diagnostics.append(
|
|
CronRunDiagnostic(
|
|
run_id=data.get("run_id", ""),
|
|
job_id=data.get("job_id", ""),
|
|
outcome=CronRunOutcome(data.get("outcome", "success")),
|
|
phases=[
|
|
CronPhaseRecord(
|
|
phase=CronAgentExecutionPhase(p["phase"]),
|
|
started_at_ms=p["started_at_ms"],
|
|
finished_at_ms=p.get("finished_at_ms"),
|
|
error=p.get("error"),
|
|
)
|
|
for p in data.get("phases", [])
|
|
],
|
|
total_duration_ms=data.get("total_duration_ms", 0),
|
|
error=data.get("error"),
|
|
token_usage=data.get("token_usage", {}),
|
|
delivery_receipts=[
|
|
CronDeliveryReceipt(
|
|
target=CronDeliveryTarget.from_dict(r["target"]),
|
|
success=r.get("success", False),
|
|
message_id=r.get("message_id"),
|
|
error=r.get("error"),
|
|
retries=r.get("retries", 0),
|
|
sent_at_ms=r.get("sent_at_ms", 0),
|
|
duration_ms=r.get("duration_ms", 0),
|
|
)
|
|
for r in data.get("delivery_receipts", [])
|
|
],
|
|
started_at_ms=data.get("started_at_ms", 0),
|
|
finished_at_ms=data.get("finished_at_ms", 0),
|
|
)
|
|
)
|
|
except (json.JSONDecodeError, KeyError, TypeError):
|
|
logger.warning("CronEngine: failed to parse diagnostic line")
|
|
self._diagnostics = self._diagnostics[-DEFAULT_DIAGNOSTIC_KEEP:]
|
|
logger.info("CronEngine: loaded %d diagnostics from %s", len(self._diagnostics), diag_path)
|
|
except OSError:
|
|
logger.warning("CronEngine: failed to load diagnostics, starting fresh")
|
|
|
|
def _save_store(self):
|
|
try:
|
|
self._store_path.parent.mkdir(parents=True, exist_ok=True)
|
|
data = {
|
|
"version": 1,
|
|
"jobs": [self._serialize_job(j) for j in self._jobs.values()],
|
|
}
|
|
tmp_path = self._store_path.with_suffix(".tmp")
|
|
tmp_path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
os.replace(tmp_path, self._store_path)
|
|
except OSError:
|
|
logger.exception("CronEngine: failed to save store")
|
|
|
|
def _save_state(self):
|
|
try:
|
|
self._state_path.parent.mkdir(parents=True, exist_ok=True)
|
|
state_data = {
|
|
"version": 1,
|
|
"jobs": {j.id: j.state.to_dict() for j in self._jobs.values()},
|
|
}
|
|
tmp_path = self._state_path.with_suffix(".tmp")
|
|
tmp_path.write_text(json.dumps(state_data, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
os.replace(tmp_path, self._state_path)
|
|
except OSError:
|
|
logger.exception("CronEngine: failed to save state")
|
|
|
|
def _save_all(self):
|
|
self._save_store()
|
|
self._save_state()
|
|
|
|
def _serialize_job(self, job: CronJob) -> dict:
|
|
d = job.to_dict()
|
|
d.pop("handler", None)
|
|
d.pop("handler_args", None)
|
|
return d
|
|
|
|
def _deserialize_job(self, jd: dict, state_data: dict | None = None) -> CronJob:
|
|
state_dict = jd.get("state", {})
|
|
if state_data:
|
|
state_dict = {**state_dict, **state_data}
|
|
|
|
job = CronJob(
|
|
id=jd["id"],
|
|
name=jd.get("name", ""),
|
|
handler=resolve_handler(jd.get("handler_name", "")),
|
|
handler_name=jd.get("handler_name", ""),
|
|
handler_args={},
|
|
schedule_kind=ScheduleKind(jd.get("schedule_kind", "cron")),
|
|
schedule_value=jd.get("schedule_value", jd.get("cron_expr", "* * * * *")),
|
|
schedule_tz=jd.get("schedule_tz"),
|
|
stagger_ms=jd.get("stagger_ms", 0),
|
|
every_ms=jd.get("every_ms"),
|
|
anchor_ms=jd.get("anchor_ms"),
|
|
status=CronJobStatus(jd.get("status", "active")),
|
|
enabled=jd.get("enabled", True),
|
|
max_runs=jd.get("max_runs", 0),
|
|
delete_after_run=jd.get("delete_after_run", False),
|
|
error_threshold=jd.get("error_threshold", 5),
|
|
delivery=[CronDeliveryTarget.from_dict(t) for t in jd.get("delivery", []) if isinstance(t, dict)],
|
|
failure_alert=CronFailureAlert.from_dict(jd.get("failure_alert")),
|
|
state=CronJobState.from_dict(state_dict),
|
|
created_at=jd.get("created_at", time.time()),
|
|
updated_at=jd.get("updated_at", time.time()),
|
|
)
|
|
return job
|
|
|
|
def add_job(self, job: CronJob) -> None:
|
|
self._ensure_loaded()
|
|
if job.id in self._jobs:
|
|
raise ValueError(f"Job '{job.id}' already registered")
|
|
normalize_job(job)
|
|
job.state.next_run_at_ms = compute_next_run_at_ms(job)
|
|
self._jobs[job.id] = job
|
|
self._save_all()
|
|
|
|
def remove_job(self, job_id: str) -> None:
|
|
self._ensure_loaded()
|
|
self._jobs.pop(job_id, None)
|
|
self._save_all()
|
|
|
|
def get_job(self, job_id: str) -> CronJob | None:
|
|
self._ensure_loaded()
|
|
return self._jobs.get(job_id)
|
|
|
|
def list_jobs(self) -> list[CronJob]:
|
|
self._ensure_loaded()
|
|
return list(self._jobs.values())
|
|
|
|
def update_job(self, job_id: str, **kwargs) -> CronJob | None:
|
|
self._ensure_loaded()
|
|
job = self._jobs.get(job_id)
|
|
if job is None:
|
|
return None
|
|
allowed = frozenset({
|
|
"name", "enabled", "schedule_kind", "schedule_value", "schedule_tz",
|
|
"stagger_ms", "every_ms", "anchor_ms", "max_runs", "error_threshold",
|
|
"delivery", "failure_alert", "delete_after_run",
|
|
})
|
|
for k, v in kwargs.items():
|
|
if k in allowed and hasattr(job, k):
|
|
setattr(job, k, v)
|
|
job.updated_at = time.time()
|
|
normalize_job(job)
|
|
self._save_all()
|
|
return job
|
|
|
|
def pause_job(self, job_id: str) -> None:
|
|
self._ensure_loaded()
|
|
job = self._jobs.get(job_id)
|
|
if job:
|
|
job.status = CronJobStatus.PAUSED
|
|
job.updated_at = time.time()
|
|
self._save_all()
|
|
|
|
def resume_job(self, job_id: str) -> None:
|
|
self._ensure_loaded()
|
|
job = self._jobs.get(job_id)
|
|
if job:
|
|
job.status = CronJobStatus.ACTIVE
|
|
job.state.next_run_at_ms = compute_next_run_at_ms(job)
|
|
job.updated_at = time.time()
|
|
self._save_all()
|
|
|
|
async def start(self) -> None:
|
|
self._ensure_loaded()
|
|
if self.is_running:
|
|
return
|
|
self._stop_event.clear()
|
|
self._task = asyncio.create_task(self._loop(), name="cron-engine")
|
|
logger.info("CronEngine started with %d jobs", len(self._jobs))
|
|
|
|
async def stop(self) -> None:
|
|
if not self.is_running:
|
|
return
|
|
self._stop_event.set()
|
|
|
|
if self._task and not self._task.done():
|
|
self._task.cancel()
|
|
|
|
running = [t for t in self._running_tasks if not t.done()]
|
|
if running:
|
|
for task in running:
|
|
task.cancel()
|
|
try:
|
|
await asyncio.wait_for(asyncio.gather(*running, return_exceptions=True), timeout=TIMEOUT.cron.gather)
|
|
except TimeoutError:
|
|
logger.warning("CronEngine: some running jobs did not cancel within 15s")
|
|
logger.info("CronEngine cancelled %d running jobs", len(running))
|
|
|
|
if self._task:
|
|
try:
|
|
await asyncio.wait_for(self._task, timeout=TIMEOUT.cron.task)
|
|
except (asyncio.CancelledError, TimeoutError):
|
|
pass
|
|
finally:
|
|
if not self._task.done():
|
|
logger.warning("CronEngine: main task did not terminate gracefully")
|
|
self._task = None
|
|
self._running_tasks.clear()
|
|
self._save_all()
|
|
logger.info("CronEngine stopped")
|
|
|
|
async def force_run(self, job_id: str) -> dict:
|
|
self._ensure_loaded()
|
|
job = self._jobs.get(job_id)
|
|
if job is None:
|
|
return {"success": False, "error": f"Job '{job_id}' not found"}
|
|
if job.state.running_at_ms is not None:
|
|
return {"success": False, "error": f"Job '{job_id}' is already running"}
|
|
return await self._run_job(job)
|
|
|
|
async def _loop(self) -> None:
|
|
while not self._stop_event.is_set():
|
|
try:
|
|
await self._tick()
|
|
except Exception:
|
|
logger.exception("CronEngine tick error")
|
|
try:
|
|
await asyncio.wait_for(
|
|
self._stop_event.wait(),
|
|
timeout=self._tick_interval,
|
|
)
|
|
break
|
|
except TimeoutError:
|
|
pass
|
|
self._save_all()
|
|
|
|
async def _tick(self) -> None:
|
|
now_ms = int(time.time() * 1000)
|
|
due_jobs = [
|
|
j
|
|
for j in self._jobs.values()
|
|
if j.status == CronJobStatus.ACTIVE
|
|
and j.enabled
|
|
and j.state.next_run_at_ms is not None
|
|
and now_ms >= j.state.next_run_at_ms
|
|
and j.state.running_at_ms is None
|
|
]
|
|
if not due_jobs:
|
|
return
|
|
|
|
tasks = [asyncio.create_task(self._run_job(j), name=f"cron-job-{j.id}") for j in due_jobs]
|
|
self._running_tasks.update(tasks)
|
|
try:
|
|
await asyncio.wait_for(
|
|
asyncio.gather(*tasks, return_exceptions=True),
|
|
timeout=TIMEOUT.cron.delivery,
|
|
)
|
|
except TimeoutError:
|
|
logger.warning("CronEngine tick timeout: %d jobs did not complete in 300s", len(due_jobs))
|
|
for task in tasks:
|
|
if not task.done():
|
|
task.cancel()
|
|
await asyncio.gather(*tasks, return_exceptions=True)
|
|
finally:
|
|
for task in tasks:
|
|
if not task.done():
|
|
task.cancel()
|
|
try:
|
|
await task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
self._running_tasks.difference_update(tasks)
|
|
|
|
async def _run_job(self, job: CronJob) -> dict:
|
|
async with self._semaphore:
|
|
if job.state.running_at_ms is not None:
|
|
return {"success": False, "error": f"Job '{job.id}' is already running"}
|
|
|
|
if job.max_runs > 0 and job.state.run_count >= job.max_runs:
|
|
job.status = CronJobStatus.EXPIRED
|
|
self._save_all()
|
|
return {"success": False, "error": "max_runs reached"}
|
|
|
|
tracker = PhaseTracker(job)
|
|
diagnostic = tracker.diagnostic
|
|
exec_phase = tracker.start_executing()
|
|
|
|
run_start_mono = time.monotonic()
|
|
job.state.running_at_ms = diagnostic.started_at_ms
|
|
|
|
try:
|
|
result = await job.handler(**(job.handler_args or {}))
|
|
tracker.finish_executing(exec_phase)
|
|
error = None
|
|
outcome = CronRunOutcome.SUCCESS
|
|
|
|
job.state.last_run_status = CronRunStatus.OK
|
|
job.state.last_error = None
|
|
job.state.run_count += 1
|
|
job.state.consecutive_errors = 0
|
|
job.state.consecutive_skipped = 0
|
|
job.state.next_run_at_ms = compute_next_with_error_tracking(job)
|
|
job.status = CronJobStatus.ACTIVE
|
|
|
|
summ_phase = tracker.start_summarizing()
|
|
tracker.finish_summarizing(summ_phase)
|
|
|
|
deliver_phase = tracker.start_delivering()
|
|
delivery_receipts: list = []
|
|
if job.delivery and self._delivery_handler:
|
|
delivery_receipts = await deliver_result(
|
|
self._delivery_handler,
|
|
job,
|
|
result if isinstance(result, dict) else {"result": str(result)},
|
|
None,
|
|
diagnostic,
|
|
self._delivery_config,
|
|
)
|
|
tracker.finish_delivering(deliver_phase)
|
|
|
|
receipt_dicts = [r.to_dict() for r in delivery_receipts]
|
|
except Exception as e:
|
|
try:
|
|
tracker.finish_executing(exec_phase, str(e))
|
|
except Exception:
|
|
logger.exception("CronEngine: failed to finish executing tracker for job '%s'", job.id)
|
|
error = str(e)
|
|
outcome = CronRunOutcome.FAILED
|
|
|
|
job.state.error_count += 1
|
|
job.state.consecutive_errors += 1
|
|
job.state.last_run_status = CronRunStatus.ERROR
|
|
job.state.last_error = error
|
|
job.state.next_run_at_ms = compute_next_with_error_tracking(job)
|
|
|
|
if job.state.consecutive_errors >= job.error_threshold:
|
|
job.status = CronJobStatus.FAILED
|
|
|
|
deliver_phase = tracker.start_delivering()
|
|
delivery_receipts: list = []
|
|
if job.delivery and self._delivery_handler:
|
|
delivery_receipts = await deliver_result(
|
|
self._delivery_handler,
|
|
job,
|
|
None,
|
|
error,
|
|
diagnostic,
|
|
self._delivery_config,
|
|
)
|
|
tracker.finish_delivering(deliver_phase)
|
|
receipt_dicts = [r.to_dict() for r in delivery_receipts]
|
|
result = None
|
|
|
|
logger.exception(
|
|
"CronEngine job '%s' failed (consecutive: %d)",
|
|
job.id,
|
|
job.state.consecutive_errors,
|
|
)
|
|
|
|
duration_ms = int((time.monotonic() - run_start_mono) * 1000)
|
|
token_usage = _extract_token_usage(result) if not error else {}
|
|
diagnostic = tracker.close(outcome, error, token_usage)
|
|
self._append_diagnostic(diagnostic)
|
|
|
|
job.state.last_run_at_ms = diagnostic.started_at_ms
|
|
job.state.last_duration_ms = duration_ms
|
|
job.state.running_at_ms = None
|
|
job.updated_at = time.time()
|
|
|
|
self._append_run_log(
|
|
job,
|
|
{
|
|
"ts": diagnostic.started_at_ms,
|
|
"job_id": job.id,
|
|
"action": "finished",
|
|
"status": "ok" if outcome == CronRunOutcome.SUCCESS else "error",
|
|
"run_id": diagnostic.run_id,
|
|
"run_at_ms": diagnostic.started_at_ms,
|
|
"duration_ms": duration_ms,
|
|
"next_run_at_ms": job.state.next_run_at_ms,
|
|
"consecutive_errors": job.state.consecutive_errors,
|
|
**({"error": error} if error else {}),
|
|
**(
|
|
{"result": result if isinstance(result, dict) else {"result": str(result)}}
|
|
if result is not None
|
|
else {}
|
|
),
|
|
"phases": [p.to_dict() for p in diagnostic.phases],
|
|
"delivery": receipt_dicts,
|
|
},
|
|
)
|
|
|
|
if job.delete_after_run and job.schedule_kind == ScheduleKind.AT:
|
|
self._jobs.pop(job.id, None)
|
|
logger.info("CronEngine: deleted one-shot job '%s'", job.id)
|
|
|
|
await evaluate_alerts(self._delivery_handler, job)
|
|
self._save_all()
|
|
|
|
if outcome == CronRunOutcome.SUCCESS:
|
|
return {
|
|
"success": True,
|
|
"job_id": job.id,
|
|
"run_id": diagnostic.run_id,
|
|
"diagnostic": diagnostic.to_dict(),
|
|
}
|
|
return {
|
|
"success": False,
|
|
"error": error,
|
|
"run_id": diagnostic.run_id,
|
|
"diagnostic": diagnostic.to_dict(),
|
|
}
|
|
|
|
def _append_run_log(self, job: CronJob, entry: dict):
|
|
try:
|
|
log_path = self._run_log_path(job.id)
|
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
line = json.dumps(entry, ensure_ascii=False) + "\n"
|
|
with open(log_path, "a", encoding="utf-8") as f:
|
|
f.write(line)
|
|
self._prune_run_log(log_path)
|
|
except OSError:
|
|
logger.exception("CronEngine: failed to write run log for '%s'", job.id)
|
|
|
|
def _prune_run_log(self, log_path: Path):
|
|
stat = log_path.stat()
|
|
if stat.st_size <= DEFAULT_RUN_LOG_MAX_BYTES:
|
|
return
|
|
try:
|
|
lines = log_path.read_text(encoding="utf-8").strip().split("\n")
|
|
kept = lines[-DEFAULT_RUN_LOG_KEEP_LINES:]
|
|
log_path.write_text("\n".join(kept) + "\n", encoding="utf-8")
|
|
except OSError:
|
|
pass
|
|
|
|
def read_run_log(self, job_id: str, limit: int = 50, offset: int = 0) -> dict:
|
|
log_path = self._run_log_path(job_id)
|
|
if not log_path.exists():
|
|
return {"entries": [], "total": 0, "offset": offset, "limit": limit, "has_more": False}
|
|
try:
|
|
raw = log_path.read_text(encoding="utf-8")
|
|
lines = [json.loads(line) for line in raw.strip().split("\n") if line.strip()]
|
|
except (OSError, json.JSONDecodeError):
|
|
return {"entries": [], "total": 0, "offset": offset, "limit": limit, "has_more": False}
|
|
|
|
total = len(lines)
|
|
start = max(0, min(total, offset))
|
|
end = min(total, start + limit)
|
|
entries = lines[start:end]
|
|
return {
|
|
"entries": entries,
|
|
"total": total,
|
|
"offset": offset,
|
|
"limit": limit,
|
|
"has_more": end < total,
|
|
}
|
|
|
|
def _append_diagnostic(self, diagnostic: CronRunDiagnostic) -> None:
|
|
self._diagnostics.append(diagnostic)
|
|
if len(self._diagnostics) > DEFAULT_DIAGNOSTIC_KEEP:
|
|
self._diagnostics = self._diagnostics[-DEFAULT_DIAGNOSTIC_KEEP:]
|
|
self._write_diagnostic_log(diagnostic)
|
|
|
|
def _write_diagnostic_log(self, diagnostic: CronRunDiagnostic) -> None:
|
|
try:
|
|
diag_path = self._diagnostic_log_path()
|
|
diag_path.parent.mkdir(parents=True, exist_ok=True)
|
|
line = json.dumps(diagnostic.to_dict(), ensure_ascii=False) + "\n"
|
|
with open(diag_path, "a", encoding="utf-8") as f:
|
|
f.write(line)
|
|
self._prune_diagnostic_log(diag_path)
|
|
except OSError:
|
|
logger.exception("CronEngine: failed to write diagnostic log")
|
|
|
|
def _prune_diagnostic_log(self, diag_path: Path) -> None:
|
|
try:
|
|
stat = diag_path.stat()
|
|
except OSError:
|
|
return
|
|
if stat.st_size <= DEFAULT_DIAGNOSTIC_LOG_MAX_BYTES:
|
|
return
|
|
try:
|
|
lines = diag_path.read_text(encoding="utf-8").strip().split("\n")
|
|
kept = lines[-DEFAULT_DIAGNOSTIC_LOG_KEEP_LINES:]
|
|
diag_path.write_text("\n".join(kept) + "\n", encoding="utf-8")
|
|
except OSError:
|
|
pass
|
|
|
|
def read_diagnostics(self, job_id: str | None = None, limit: int = 50, offset: int = 0) -> dict:
|
|
all_diags = self._read_all_diagnostics()
|
|
filtered = all_diags
|
|
if job_id:
|
|
filtered = [d for d in filtered if d.job_id == job_id]
|
|
total = len(filtered)
|
|
start = max(0, min(total, offset))
|
|
end = min(total, start + limit)
|
|
entries = [d.to_dict() for d in filtered[start:end]]
|
|
return {
|
|
"entries": entries,
|
|
"total": total,
|
|
"offset": offset,
|
|
"limit": limit,
|
|
"has_more": end < total,
|
|
}
|
|
|
|
def _read_all_diagnostics(self) -> list[CronRunDiagnostic]:
|
|
diag_path = self._diagnostic_log_path()
|
|
if not diag_path.exists():
|
|
return list(self._diagnostics)
|
|
memory_ids = {d.run_id for d in self._diagnostics if d.run_id}
|
|
result = list(self._diagnostics)
|
|
try:
|
|
raw = diag_path.read_text(encoding="utf-8")
|
|
for line in raw.strip().split("\n"):
|
|
if not line.strip():
|
|
continue
|
|
try:
|
|
data = json.loads(line)
|
|
if data.get("run_id") in memory_ids:
|
|
continue
|
|
result.append(
|
|
CronRunDiagnostic(
|
|
run_id=data.get("run_id", ""),
|
|
job_id=data.get("job_id", ""),
|
|
outcome=CronRunOutcome(data.get("outcome", "success")),
|
|
phases=[
|
|
CronPhaseRecord(
|
|
phase=CronAgentExecutionPhase(p["phase"]),
|
|
started_at_ms=p["started_at_ms"],
|
|
finished_at_ms=p.get("finished_at_ms"),
|
|
error=p.get("error"),
|
|
)
|
|
for p in data.get("phases", [])
|
|
],
|
|
total_duration_ms=data.get("total_duration_ms", 0),
|
|
error=data.get("error"),
|
|
token_usage=data.get("token_usage", {}),
|
|
delivery_receipts=[
|
|
CronDeliveryReceipt(
|
|
target=CronDeliveryTarget.from_dict(r["target"]),
|
|
success=r.get("success", False),
|
|
message_id=r.get("message_id"),
|
|
error=r.get("error"),
|
|
retries=r.get("retries", 0),
|
|
sent_at_ms=r.get("sent_at_ms", 0),
|
|
duration_ms=r.get("duration_ms", 0),
|
|
)
|
|
for r in data.get("delivery_receipts", [])
|
|
],
|
|
started_at_ms=data.get("started_at_ms", 0),
|
|
finished_at_ms=data.get("finished_at_ms", 0),
|
|
)
|
|
)
|
|
except (json.JSONDecodeError, KeyError, TypeError):
|
|
pass
|
|
except OSError:
|
|
pass
|
|
return sorted(result, key=lambda d: d.started_at_ms)
|
|
|
|
def list_diagnostics_for_job(self, job_id: str, limit: int = 10) -> list[dict]:
|
|
all_diags = self._read_all_diagnostics()
|
|
matching = [d for d in all_diags if d.job_id == job_id]
|
|
return [d.to_dict() for d in matching[-limit:]]
|
|
|
|
|
|
def _extract_token_usage(result: dict | None) -> dict:
|
|
if not result or not isinstance(result, dict):
|
|
return {}
|
|
usage = {}
|
|
for key in TOKEN_USAGE_KEYS:
|
|
if key in result and isinstance(result[key], (int, float)):
|
|
usage[key] = result[key]
|
|
for nested_key in TOKEN_USAGE_NESTED_KEYS:
|
|
if nested_key in result and isinstance(result[nested_key], dict):
|
|
for k, v in result[nested_key].items():
|
|
if isinstance(v, (int, float)):
|
|
usage[k] = v
|
|
return usage
|
|
|
|
|
|
_engine: CronEngine | None = None
|
|
|
|
|
|
def get_cron_engine() -> CronEngine:
|
|
global _engine
|
|
if _engine is None:
|
|
_engine = CronEngine()
|
|
return _engine
|