Kris
56022a9199
- 修复 scheduler store 末尾多余逗号 - 将智能体名称从“语析”改为“Kris” - 新增获取运行记录UID的权限校验接口 - 重构操作日志写入逻辑，使用独立会话避免事务污染 - 注册外部系统调度处理器 - 优化全局错误处理器，统一响应格式与序列化处理 - 新增调度任务运行日志的handler_name字段与索引 - 重构任务执行函数，新增payload覆盖与操作人审计参数 - 重写外部系统store，新增侧边栏折叠状态与持久化 - 优化会话、消息等模型的索引、约束与字段类型 - 新增多项配置项与环境变量支持 - 重构渠道相关模型，新增索引、约束与字段优化 - 新增渠道插件模型与内容审核模型的完善
426 lines
18 KiB
Python
426 lines
18 KiB
Python
"""定时任务调度相关的 PostgreSQL 数据模型。
|
||
|
||
表名统一以 `scheduled_task_` 为前缀；属于调度限界上下文（scheduler bounded
|
||
context），与 `external_systems` / `channel` 等上下文隔离，不跨上下文 import
|
||
实现类。字段类型复用 `JSON().with_variant(JSONB, "postgresql")` 以兼容多方言。
|
||
|
||
本模块仅承载调度域自身的持久化模型：
|
||
- `scheduled_tasks`：任务定义与状态机
|
||
- `scheduled_task_run_logs`：每次执行的审计日志
|
||
- `scheduled_task_run_log_daily`：按天 + handler_name 维度的执行统计聚合
|
||
（借鉴 xxl-job `xxl_job_logreport` 分离设计，统计查询走聚合表避免扫描明细）
|
||
- `scheduled_task_idempotency`：Admin API 写操作幂等去重
|
||
|
||
注意：`task_id` 在子表中为字符串引用而非外键（PRD §6.1.6），允许 `at` 类型
|
||
任务删除后日志保留以维持审计轨迹。所有表统一包含审计字段
|
||
（created_by/updated_by/created_at/updated_at/is_deleted/deleted_at），
|
||
支持软删除与操作追溯，与 `external_systems` / `channel` 等上下文保持一致。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import Any
|
||
|
||
from sqlalchemy import (
|
||
DATE,
|
||
JSON,
|
||
Boolean,
|
||
CheckConstraint,
|
||
Column,
|
||
DateTime,
|
||
Index,
|
||
Integer,
|
||
String,
|
||
Text,
|
||
UniqueConstraint,
|
||
text,
|
||
)
|
||
from sqlalchemy.dialects.postgresql import JSONB
|
||
from yuxi.storage.postgres.models_business import Base
|
||
from yuxi.utils.datetime_utils import format_utc_datetime, utc_now_naive
|
||
|
||
JSON_VALUE = JSON().with_variant(JSONB, "postgresql")
|
||
|
||
|
||
class ScheduledTask(Base):
|
||
"""定时任务定义。
|
||
|
||
一条记录对应一次任务注册；`next_run_at` 由调度器在创建/执行后按
|
||
`schedule_kind`（cron / at）计算。`status` 表示任务状态机当前所处状态
|
||
（active / paused / dead_letter），`enabled` 与 `status` 通过应用层联动
|
||
（见 ``scheduler_service.update_task`` 方案 E）：``enabled=False`` 且
|
||
``status=active`` 同步置 ``paused``；``enabled=True`` 且 ``status=paused``
|
||
同步置 ``active``。tick 扫描条件为 ``enabled=True AND status='active'
|
||
AND next_run_at <= now``，两者共同决定可执行性。
|
||
"""
|
||
|
||
__tablename__ = "scheduled_tasks"
|
||
|
||
# 主键
|
||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||
|
||
# 业务标识
|
||
task_id = Column(
|
||
String(64),
|
||
nullable=False,
|
||
comment="业务 UUID；system 任务使用约定前缀如 system:run_log_cleanup；"
|
||
"唯一性由部分唯一索引 uq_scheduled_tasks_task_id_active 保障（is_deleted=0），"
|
||
"软删除后允许重建",
|
||
)
|
||
handler_name = Column(
|
||
String(128), nullable=False, index=True, comment="引用已注册 handler（非外键，worker 执行时校验）"
|
||
)
|
||
|
||
# 归属范围（v1 仅作数据隔离字段，不引入 business_admin 角色）
|
||
owner_scope = Column(String(32), nullable=False, default="system", comment="任务归属范围：system / business")
|
||
owner_id = Column(String(128), nullable=False, comment="业务方 ID，如 channel_type:account_id")
|
||
|
||
# 调度配置
|
||
schedule_kind = Column(String(16), nullable=False, default="cron", comment="调度类型：cron / at")
|
||
cron_expression = Column(String(64), nullable=True, comment="5 字段标准 cron 表达式；cron 类型必填")
|
||
run_at = Column(
|
||
DateTime,
|
||
nullable=True,
|
||
comment="一次性执行时间（UTC，与 tz 无关）；at 类型必填且必须未来时间。"
|
||
"at 类型创建时 next_run_at 同步设为此值，执行后 next_run_at 置 NULL",
|
||
)
|
||
tz = Column(
|
||
String(64),
|
||
nullable=False,
|
||
default="Asia/Shanghai",
|
||
comment="IANA 时区，仅用于 cron 下一次执行时间计算；run_at 始终以 UTC 存储，不受 tz 影响",
|
||
)
|
||
payload = Column(JSON_VALUE, nullable=False, default=dict, comment="handler 入参（≤64KB）")
|
||
|
||
# 运行时控制
|
||
enabled = Column(
|
||
Boolean,
|
||
nullable=False,
|
||
default=True,
|
||
comment="是否启用；与 status 通过应用层联动（方案 E）："
|
||
"enabled=False 且 status=active 同步置 paused；"
|
||
"dead_letter 任务需通过 resume_task 恢复，不能通过 enabled=True 直接启用",
|
||
)
|
||
delete_after_run = Column(
|
||
Boolean,
|
||
nullable=False,
|
||
default=False,
|
||
comment="执行成功后是否删除任务记录；仅对 at 类型有意义，"
|
||
"cron 类型应用层拒绝设置为 True；DB 默认 False，at 类型由应用层显式设置 True",
|
||
)
|
||
block_strategy = Column(
|
||
String(32),
|
||
nullable=False,
|
||
default="discard_later",
|
||
comment="阻塞处理策略：discard_later（v1）/ serial_execution（v2）/ "
|
||
"cover_early（v2）；借鉴 xxl-job executor_block_strategy",
|
||
)
|
||
stagger_seconds = Column(Integer, nullable=True, comment="整点抖动秒数；仅 cron 整点模式计算，按 task_id 哈希固定")
|
||
|
||
# 状态机
|
||
consecutive_errors = Column(
|
||
Integer, nullable=False, default=0, comment="连续失败次数；成功执行后归零，达阈值进入死信"
|
||
)
|
||
status = Column(
|
||
String(32), nullable=False, default="active", index=True, comment="任务状态：active / paused / dead_letter"
|
||
)
|
||
|
||
# 执行轨迹
|
||
last_run_at = Column(
|
||
DateTime,
|
||
nullable=True,
|
||
comment="上次执行开始时间（方案 B：语义统一为执行开始时间，"
|
||
"由 record_run_result 的 started_at 参数写入，非执行结束时间）",
|
||
)
|
||
next_run_at = Column(
|
||
DateTime,
|
||
nullable=True,
|
||
comment="下次执行时间（UTC）。cron 类型由调度器按 cron 表达式计算；"
|
||
"at 类型创建时=run_at，成功执行后置 NULL（方案 A：通过 "
|
||
"finalize_one_shot_task 置 NULL + status=paused 避免无限循环）；"
|
||
"失败退避时按退避序列计算",
|
||
)
|
||
last_error = Column(Text, nullable=True, comment="最近一次错误信息")
|
||
|
||
# 审计字段
|
||
created_by = Column(String(64), nullable=True)
|
||
updated_by = Column(String(64), nullable=True)
|
||
created_at = Column(DateTime, nullable=False, default=utc_now_naive)
|
||
updated_at = Column(DateTime, nullable=False, default=utc_now_naive, onupdate=utc_now_naive)
|
||
is_deleted = Column(Integer, nullable=False, default=0, index=True)
|
||
deleted_at = Column(DateTime, nullable=True)
|
||
|
||
__table_args__ = (
|
||
# A1: cron/at 互斥约束 —— cron 必填 cron_expression，at 必填 run_at
|
||
CheckConstraint(
|
||
"(schedule_kind = 'cron' AND cron_expression IS NOT NULL) OR (schedule_kind = 'at' AND run_at IS NOT NULL)",
|
||
name="ck_scheduled_tasks_schedule_kind_fields",
|
||
),
|
||
# B1: tick 扫描索引 —— WHERE enabled=True AND status='active' AND next_run_at <= ?
|
||
Index("ix_scheduled_tasks_tick_scan", "enabled", "status", "next_run_at"),
|
||
Index("ix_scheduled_tasks_owner", "owner_scope", "owner_id"),
|
||
# A2: task_id 部分唯一索引 —— 软删除后允许重建同名 task_id
|
||
Index(
|
||
"uq_scheduled_tasks_task_id_active",
|
||
"task_id",
|
||
unique=True,
|
||
postgresql_where=text("is_deleted = 0"),
|
||
),
|
||
# 列表查询复合索引：常见路径 WHERE is_deleted=0 ORDER BY created_at DESC
|
||
# 覆盖软删除过滤 + 按创建时间排序，避免 filesort
|
||
Index("ix_scheduled_tasks_list", "is_deleted", "created_at"),
|
||
# 回收站查询索引：WHERE is_deleted=1 ORDER BY deleted_at DESC
|
||
Index("ix_scheduled_tasks_recycle", "is_deleted", "deleted_at"),
|
||
# 工作台状态计数与异常查询部分索引：WHERE is_deleted=0
|
||
# 覆盖 count_by_status 的 GROUP BY status 及异常任务按 status 过滤，
|
||
# 部分索引仅包含未删除行，体积小且命中率高
|
||
Index(
|
||
"ix_scheduled_tasks_status_alive",
|
||
"status",
|
||
"consecutive_errors",
|
||
postgresql_where=text("is_deleted = 0"),
|
||
),
|
||
)
|
||
|
||
def to_dict(self) -> dict[str, Any]:
|
||
"""序列化为 API 响应字典。"""
|
||
return {
|
||
"id": self.id,
|
||
"task_id": self.task_id,
|
||
"handler_name": self.handler_name,
|
||
"owner_scope": self.owner_scope,
|
||
"owner_id": self.owner_id,
|
||
"schedule_kind": self.schedule_kind,
|
||
"cron_expression": self.cron_expression,
|
||
"run_at": format_utc_datetime(self.run_at) if self.run_at else None,
|
||
"tz": self.tz,
|
||
"payload": self.payload or {},
|
||
"enabled": bool(self.enabled),
|
||
"delete_after_run": bool(self.delete_after_run),
|
||
"block_strategy": self.block_strategy,
|
||
"stagger_seconds": self.stagger_seconds,
|
||
"consecutive_errors": self.consecutive_errors,
|
||
"status": self.status,
|
||
"last_run_at": format_utc_datetime(self.last_run_at) if self.last_run_at else None,
|
||
"next_run_at": format_utc_datetime(self.next_run_at) if self.next_run_at else None,
|
||
"last_error": self.last_error,
|
||
"created_by": self.created_by,
|
||
"updated_by": self.updated_by,
|
||
"created_at": format_utc_datetime(self.created_at),
|
||
"updated_at": format_utc_datetime(self.updated_at),
|
||
"is_deleted": self.is_deleted,
|
||
"deleted_at": format_utc_datetime(self.deleted_at) if self.deleted_at else None,
|
||
}
|
||
|
||
|
||
class ScheduledTaskRunLog(Base):
|
||
"""定时任务单次执行日志。
|
||
|
||
每次 tick 或手动触发生成一条 `running` 记录，执行结束后更新为
|
||
`success` / `failure` / `timeout`。阻塞丢弃场景（`block_strategy=
|
||
discard_later` 且上次未完成）写 `skipped`，不计入日聚合统计。
|
||
`run_id` 全局唯一，用于幂等控制与 ARQ `_job_id` 关联。`task_id` 为
|
||
字符串引用非外键，任务删除后日志保留。
|
||
"""
|
||
|
||
__tablename__ = "scheduled_task_run_logs"
|
||
|
||
# 主键
|
||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||
|
||
# 关联（字符串引用，非外键）
|
||
task_id = Column(String(64), nullable=False, comment="关联 scheduled_tasks.task_id；非外键，任务删除后日志保留")
|
||
handler_name = Column(
|
||
String(128),
|
||
nullable=True,
|
||
index=True,
|
||
comment="handler 名称冗余字段（写入时从 task 快照）；nullable 兼容存量数据，"
|
||
"支持跨任务按 handler 过滤与 reclaim RETURNING 补录日聚合统计",
|
||
)
|
||
run_id = Column(
|
||
String(64), nullable=False, unique=True, comment="单次执行唯一标识（UUID），用于幂等控制与 ARQ _job_id 关联"
|
||
)
|
||
|
||
# 执行上下文
|
||
triggered_by = Column(
|
||
String(16), nullable=False, default="auto", comment="触发方式：auto（tick）/ manual（Admin API）"
|
||
)
|
||
status = Column(
|
||
String(16),
|
||
nullable=False,
|
||
default="running",
|
||
index=True,
|
||
comment="执行状态：running / success / failure / timeout / skipped（阻塞丢弃，不计入聚合）",
|
||
)
|
||
|
||
# 执行结果
|
||
error_message = Column(Text, nullable=True, comment="失败或超时时的错误信息")
|
||
output = Column(JSON_VALUE, nullable=True, comment="handler 返回的 output")
|
||
|
||
# 时间轨迹
|
||
started_at = Column(DateTime, nullable=False, default=utc_now_naive, comment="执行开始时间")
|
||
finished_at = Column(DateTime, nullable=True, comment="执行结束时间；running 状态时为 NULL")
|
||
|
||
# 审计字段
|
||
created_by = Column(String(64), nullable=True)
|
||
updated_by = Column(String(64), nullable=True)
|
||
created_at = Column(DateTime, nullable=False, default=utc_now_naive)
|
||
updated_at = Column(DateTime, nullable=False, default=utc_now_naive, onupdate=utc_now_naive)
|
||
is_deleted = Column(Integer, nullable=False, default=0, index=True)
|
||
deleted_at = Column(DateTime, nullable=True)
|
||
|
||
__table_args__ = (
|
||
Index("ix_scheduled_task_run_logs_task_started", "task_id", "started_at"),
|
||
Index("ix_scheduled_task_run_logs_started", "started_at"),
|
||
# 复合索引：健康检查 / reclaim / list_by_status 均按 status + started_at 过滤
|
||
Index("ix_scheduled_task_run_logs_status_started", "status", "started_at"),
|
||
)
|
||
|
||
def to_dict(self) -> dict[str, Any]:
|
||
"""序列化为 API 响应字典。"""
|
||
return {
|
||
"id": self.id,
|
||
"task_id": self.task_id,
|
||
"handler_name": self.handler_name,
|
||
"run_id": self.run_id,
|
||
"triggered_by": self.triggered_by,
|
||
"status": self.status,
|
||
"error_message": self.error_message,
|
||
"output": self.output,
|
||
"started_at": format_utc_datetime(self.started_at),
|
||
"finished_at": format_utc_datetime(self.finished_at) if self.finished_at else None,
|
||
"created_by": self.created_by,
|
||
"updated_by": self.updated_by,
|
||
"created_at": format_utc_datetime(self.created_at),
|
||
"updated_at": format_utc_datetime(self.updated_at),
|
||
"is_deleted": self.is_deleted,
|
||
"deleted_at": format_utc_datetime(self.deleted_at) if self.deleted_at else None,
|
||
}
|
||
|
||
|
||
class ScheduledTaskRunLogDaily(Base):
|
||
"""定时任务执行日志日聚合统计。
|
||
|
||
借鉴 xxl-job `xxl_job_logreport` 分离设计：明细日志（`scheduled_task_run_logs`）
|
||
写多读少，按时间范围聚合查询慢；本表按天 + handler_name 维度预聚合，统计
|
||
查询走本表（90 天 × handler 数 ≈ 几百行），性能提升 2-3 个数量级。
|
||
|
||
写入时机：每次任务执行完成（success/failure/timeout）或进入死信时，由
|
||
仓储层原子 upsert（`ON CONFLICT (stat_date, handler_name) DO UPDATE`）。
|
||
`skipped` 状态（阻塞丢弃）不计入聚合。保留 1 年，由 `run_log_cleanup`
|
||
handler 顺带清理。
|
||
"""
|
||
|
||
__tablename__ = "scheduled_task_run_log_daily"
|
||
|
||
# 主键
|
||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||
|
||
# 聚合维度
|
||
stat_date = Column(DATE, nullable=False, comment="统计日期（按 tz=Asia/Shanghai 的自然日切分）")
|
||
handler_name = Column(
|
||
String(128),
|
||
nullable=False,
|
||
index=True,
|
||
comment="handler 名称；与 scheduled_tasks.handler_name 引用一致",
|
||
)
|
||
|
||
# 聚合计数
|
||
success_count = Column(Integer, nullable=False, default=0, comment="当日成功次数")
|
||
failure_count = Column(Integer, nullable=False, default=0, comment="当日失败次数")
|
||
timeout_count = Column(Integer, nullable=False, default=0, comment="当日超时次数")
|
||
dead_letter_count = Column(Integer, nullable=False, default=0, comment="当日进入死信的次数")
|
||
|
||
# 审计字段
|
||
created_by = Column(String(64), nullable=True)
|
||
updated_by = Column(String(64), nullable=True)
|
||
created_at = Column(DateTime, nullable=False, default=utc_now_naive)
|
||
updated_at = Column(DateTime, nullable=False, default=utc_now_naive, onupdate=utc_now_naive)
|
||
is_deleted = Column(Integer, nullable=False, default=0, index=True)
|
||
deleted_at = Column(DateTime, nullable=True)
|
||
|
||
__table_args__ = (
|
||
UniqueConstraint("stat_date", "handler_name", name="uq_scheduled_task_run_log_daily_date_handler"),
|
||
)
|
||
|
||
def to_dict(self) -> dict[str, Any]:
|
||
"""序列化为 API 响应字典。"""
|
||
return {
|
||
"id": self.id,
|
||
"stat_date": self.stat_date.isoformat() if self.stat_date else None,
|
||
"handler_name": self.handler_name,
|
||
"success_count": self.success_count,
|
||
"failure_count": self.failure_count,
|
||
"timeout_count": self.timeout_count,
|
||
"dead_letter_count": self.dead_letter_count,
|
||
"created_by": self.created_by,
|
||
"updated_by": self.updated_by,
|
||
"created_at": format_utc_datetime(self.created_at),
|
||
"updated_at": format_utc_datetime(self.updated_at),
|
||
"is_deleted": self.is_deleted,
|
||
"deleted_at": format_utc_datetime(self.deleted_at) if self.deleted_at else None,
|
||
}
|
||
|
||
|
||
class ScheduledTaskIdempotency(Base):
|
||
"""Admin API 写操作幂等记录。
|
||
|
||
客户端通过 `Idempotency-Key` 请求头传入唯一 key，服务端利用
|
||
`idempotency_key` UNIQUE 约束实现首次写入抢占；重复请求回放
|
||
`response_body`。记录保留 24 小时后由 `idempotency_cleanup` handler 清理。
|
||
"""
|
||
|
||
__tablename__ = "scheduled_task_idempotency"
|
||
|
||
# 主键
|
||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||
|
||
# 幂等键
|
||
idempotency_key = Column(
|
||
String(128), nullable=False, unique=True, comment="客户端传入的 Idempotency-Key，建议 UUID"
|
||
)
|
||
|
||
# 关联（字符串引用，非外键）
|
||
task_id = Column(String(64), nullable=False, comment="关联 scheduled_tasks.task_id；非外键")
|
||
|
||
# 操作信息
|
||
operation = Column(
|
||
String(32),
|
||
nullable=False,
|
||
comment="操作类型：create / update / delete / pause / resume / trigger",
|
||
)
|
||
response_body = Column(JSON_VALUE, nullable=True, comment="首次请求的响应体；用于重复请求重放")
|
||
|
||
# 审计字段
|
||
created_by = Column(String(64), nullable=True, comment="发起幂等请求的操作人 UID")
|
||
updated_by = Column(String(64), nullable=True, comment="回写响应体时的操作人 UID")
|
||
created_at = Column(
|
||
DateTime, nullable=False, default=utc_now_naive, index=True, comment="创建时间；用于清理过期记录"
|
||
)
|
||
updated_at = Column(DateTime, nullable=False, default=utc_now_naive, onupdate=utc_now_naive)
|
||
is_deleted = Column(Integer, nullable=False, default=0, index=True)
|
||
deleted_at = Column(DateTime, nullable=True)
|
||
|
||
__table_args__ = (
|
||
CheckConstraint(
|
||
"operation IN ('create', 'update', 'delete', 'pause', 'resume', 'trigger')",
|
||
name="ck_scheduled_task_idempotency_operation",
|
||
),
|
||
)
|
||
|
||
def to_dict(self) -> dict[str, Any]:
|
||
"""序列化为 API 响应字典。"""
|
||
return {
|
||
"id": self.id,
|
||
"idempotency_key": self.idempotency_key,
|
||
"task_id": self.task_id,
|
||
"operation": self.operation,
|
||
"response_body": self.response_body,
|
||
"created_by": self.created_by,
|
||
"updated_by": self.updated_by,
|
||
"created_at": format_utc_datetime(self.created_at),
|
||
"updated_at": format_utc_datetime(self.updated_at),
|
||
"is_deleted": self.is_deleted,
|
||
"deleted_at": format_utc_datetime(self.deleted_at) if self.deleted_at else None,
|
||
}
|