mirror of
https://github.com/fastapiadmin/FastapiAdmin.git
synced 2026-10-14 05:00:10 +00:00
zhangtao
a4e8727a86
背景：仓库此前没有 pyright/basedpyright 配置，basedpyright 使用自身默认（≈recommended），
在 app/ 下产生 247 errors / 3799 warnings，其中 143 条是 reportMissingTypeArgument
（未参数化泛型）这类注解完整度噪音，会把真缺陷淹没（例如 OSS 的 Client.uploader 方法误用）。
配置（backend/pyrightconfig.json）
- typeCheckingMode: basic；显式把能抓真缺陷的规则保持为 error
（reportAttributeAccessIssue / reportArgumentType / reportOptionalMemberAccess /
reportIncompatibleMethodOverride / reportConstantRedefinition / reportMissingImports 等）
- 关闭注解完整度与风格类规则；tests/ 用 executionEnvironments 单独放宽（extraPaths 保证 app 可解析）
- 结果：app + main.py + tests 全量 0 errors
- 红绿验证：合成「str | None 传参」「MethodType 上取属性」「Optional 成员访问」三类
仍全部报 error，证明校准没有把真缺陷一起关掉
修复配置暴露的三处真缺陷
1) template_safety.py：_SQLALCHEMY_TYPE_WHITELIST 对全大写常量二次赋值 → 合并为一次成型
2) template_safety.py：normalize_column_type 假定 _TYPE_BASE_RE 必命中（用 # type: ignore 盖住，
None 时 .group() 会崩）→ 改为 None 安全，去掉 type-ignore
3) node/service.py ×2：CustomException(msg=reason) 中 reason 为 str | None，客户端可能收到空文案
→ 新增 SchedulerUtil.require_schedulable() 收敛「判定 + 抛异常 + 兜底文案」，调用点改为一行
验证：basedpyright 0 errors（原 247）；ruff All checks passed；pytest 97 passed；
已部署并线上复验（四端 200、Host 收窄、伪造令牌被拒、验证码过期 400 保留业务文案、
容器内 require_schedulable 与 OSS 调用修正均已生效）
806 lines
35 KiB
Python
806 lines
35 KiB
Python
import asyncio
|
||
import json
|
||
import re
|
||
from collections.abc import Callable
|
||
from datetime import datetime
|
||
from typing import Any
|
||
|
||
from apscheduler.events import (
|
||
EVENT_ALL,
|
||
EVENT_ALL_JOBS_REMOVED,
|
||
EVENT_JOB_ADDED,
|
||
EVENT_JOB_ERROR,
|
||
EVENT_JOB_EXECUTED,
|
||
EVENT_JOB_MISSED,
|
||
EVENT_JOB_REMOVED,
|
||
EVENT_JOB_SUBMITTED,
|
||
JobEvent,
|
||
)
|
||
from apscheduler.executors.asyncio import AsyncIOExecutor
|
||
from apscheduler.executors.pool import ProcessPoolExecutor, ThreadPoolExecutor
|
||
from apscheduler.job import Job
|
||
from apscheduler.jobstores.memory import MemoryJobStore
|
||
from apscheduler.jobstores.redis import RedisJobStore
|
||
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
|
||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||
from apscheduler.triggers.cron import CronTrigger
|
||
from apscheduler.triggers.date import DateTrigger
|
||
from apscheduler.triggers.interval import IntervalTrigger
|
||
from redis.asyncio import Redis
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.config.setting import settings
|
||
from app.core.database import engine
|
||
from app.core.exceptions import CustomException
|
||
from app.core.logger import logger
|
||
|
||
# 任务状态常量（与 task_job.status 注释保持一致：0待执行 1执行中 2成功 3失败）
|
||
JOB_STATUS_SUCCESS = 2
|
||
JOB_STATUS_FAILED = 3
|
||
|
||
# 常见执行失败模式的中文结论（匹配原始异常文本；任意第三方异常无法机翻，保留原文供诊断）
|
||
_JOB_ERROR_HINTS: list[tuple[str, str]] = [
|
||
("maximum number of running instances", "任务已达最大并发实例数，本次执行被跳过（可调大节点的 max_instances）"),
|
||
("was missed by", "任务错过计划执行时间（misfire，通常因进程繁忙或重启空窗）"),
|
||
("Insufficient Balance", "AI 模型账户余额不足"),
|
||
("Payment Required", "AI 模型服务计费失败（HTTP 402）"),
|
||
("Unauthorized", "上游认证失败（HTTP 401），请检查访问令牌"),
|
||
("rate limit", "触发上游限流（HTTP 429），请稍后重试"),
|
||
("Connection", "网络连接异常，请检查目标地址与网络"),
|
||
("Timeout", "请求超时"),
|
||
]
|
||
|
||
|
||
def _humanize_job_error(detail: Any) -> str:
|
||
"""把执行异常转成"中文结论｜原始信息"回显；未命中已知模式时原样返回。"""
|
||
text = str(detail)[:4000]
|
||
for pattern, hint in _JOB_ERROR_HINTS:
|
||
if pattern in text:
|
||
return f"{hint}｜原始信息: {text}"[:4000]
|
||
return text
|
||
|
||
# 多 worker 下单实例调度锁：所有进程共享同一个 RedisJobStore，若每个进程都自行
|
||
# start，同一任务会被重复调度执行。仅持有锁的进程运行调度器并周期续期，其余进程
|
||
# 周期争抢——持有者崩溃（锁过期）后自动接管。
|
||
SCHEDULER_LOCK_KEY = "fastapiadmin:scheduler:lock"
|
||
SCHEDULER_LOCK_TTL = 30 # 锁有效期（秒）；须大于续期间隔，否则锁在续期前就过期
|
||
SCHEDULER_RENEW_INTERVAL = 10 # 持有者续期间隔
|
||
SCHEDULER_POLL_INTERVAL = 5 # 候选进程争抢间隔；越小接管越快，Redis 压力越大
|
||
|
||
scheduler = AsyncIOScheduler()
|
||
scheduler.configure(
|
||
jobstores={
|
||
"default": RedisJobStore(
|
||
host=settings.REDIS_HOST,
|
||
port=int(settings.REDIS_PORT),
|
||
username=settings.REDIS_USER or None,
|
||
password=settings.REDIS_PASSWORD or None,
|
||
db=int(settings.REDIS_DB_NAME),
|
||
),
|
||
"sqlalchemy": SQLAlchemyJobStore(url=settings.DB_URI, engine=engine),
|
||
"memory": MemoryJobStore(),
|
||
},
|
||
executors={
|
||
"default": AsyncIOExecutor(),
|
||
"threadpool": ThreadPoolExecutor(max_workers=10),
|
||
"processpool": ProcessPoolExecutor(max_workers=1),
|
||
},
|
||
job_defaults={
|
||
"coalesce": True,
|
||
"max_instances": 5,
|
||
},
|
||
timezone="Asia/Shanghai",
|
||
)
|
||
|
||
|
||
class SchedulerUtil:
|
||
"""定时任务 SDK — 封装 APScheduler 核心操作与任务执行记录落库。"""
|
||
|
||
redis_instance: Redis | None = None
|
||
# ── 内置处理器白名单（「脚本类」与「内置函数类」任务的分界）────────────────
|
||
# 节点任务 func 形如 "builtin:<模块名>[.<函数名>]"（模块限定在 handlers 包内）时，
|
||
# 由注册表解析后**直接调用**，不经过 exec —— 因此不受 SCHEDULER_ALLOW_CODE_EXEC 限制。
|
||
# 这样该开关只关闭"执行用户提交代码块"的能力，而不是整个调度特性。
|
||
BUILTIN_PREFIX = "builtin:"
|
||
BUILTIN_HANDLERS_MODULE = "app.modules.task.cronjob.node.handlers"
|
||
# 名称白名单：仅允许「模块名」或「模块名.函数名」，禁止点号穿越 / 括号 / 特殊字符
|
||
_BUILTIN_NAME_RE = re.compile(r"^[a-z_][a-z0-9_]*(\.[a-z_][a-z0-9_]*)?$")
|
||
_builtin_handlers: dict[str, Callable[..., Any]] = {}
|
||
|
||
# 多 worker 选主状态：_redis 为空视为单机直跑；_leader_token 非空表示本进程持有调度锁
|
||
_redis: Redis | None = None
|
||
_leader_token: str | None = None
|
||
_maintain_task: asyncio.Task | None = None
|
||
_reconcile_task: asyncio.Task | None = None
|
||
|
||
# ------------------------------------------------------------------ #
|
||
# 生命周期（多 worker 下仅持有分布式锁的进程真正运行调度器）
|
||
# ------------------------------------------------------------------ #
|
||
@classmethod
|
||
async def init_scheduler(cls, redis: Redis | None = None) -> None:
|
||
"""应用启动时初始化定时任务调度器。
|
||
|
||
多 worker 部署（WORKERS>1）时各进程共享 RedisJobStore，若各自 start，
|
||
同一任务会被多个进程重复执行。这里用分布式锁选主：本进程抢到锁就启动
|
||
调度器；抢不到则转后台候选，持有者异常退出后自动接管。
|
||
Redis 不可用（None）时退化为本进程直接运行，等价于单机行为。
|
||
|
||
参数:
|
||
- redis (Redis | None): Redis 连接，用于调度锁与任务持久化。
|
||
|
||
返回:
|
||
- None
|
||
"""
|
||
try:
|
||
if redis:
|
||
cls.redis_instance = redis
|
||
cls._redis = redis
|
||
if await cls._ensure_scheduler_running():
|
||
logger.info("✅ 本进程持有调度器锁，定时任务调度器已启动")
|
||
else:
|
||
logger.info("🔄 调度器由其它进程持有，本进程转为候选（异常退出后自动接管）")
|
||
except Exception as e:
|
||
logger.error(f"❌ 定时任务调度器初始化失败: {e}")
|
||
raise
|
||
|
||
@classmethod
|
||
async def _ensure_scheduler_running(cls) -> bool:
|
||
"""确保调度器在运行：抢到锁则本地启动并登记后台维护协程；否则登记候选协程。
|
||
|
||
返回:
|
||
- bool: 本进程是否真正运行着调度器。
|
||
"""
|
||
if scheduler.running:
|
||
return True
|
||
if cls._redis is None:
|
||
cls._start_scheduler_local()
|
||
cls._spawn_reconcile()
|
||
return True
|
||
from app.core.redis_crud import RedisCURD
|
||
|
||
acquired, token = await RedisCURD(cls._redis).lock(key=SCHEDULER_LOCK_KEY, expire=SCHEDULER_LOCK_TTL)
|
||
if acquired:
|
||
cls._leader_token = token
|
||
cls._start_scheduler_local()
|
||
cls._spawn_maintain()
|
||
if acquired:
|
||
cls._spawn_reconcile()
|
||
return acquired
|
||
|
||
@classmethod
|
||
async def _maintain_loop(cls) -> None:
|
||
"""后台维护循环：持有者周期续期锁，锁丢失（崩溃/长时间阻塞）则让位转候选。
|
||
|
||
候选进程周期争抢锁，抢到即接管调度器。
|
||
"""
|
||
if cls._redis is None:
|
||
return
|
||
from app.core.redis_crud import RedisCURD
|
||
|
||
crud = RedisCURD(cls._redis)
|
||
try:
|
||
while True:
|
||
if cls._leader_token:
|
||
await asyncio.sleep(SCHEDULER_RENEW_INTERVAL)
|
||
renewed = await crud.renew_lock(key=SCHEDULER_LOCK_KEY, expire=SCHEDULER_LOCK_TTL, value=cls._leader_token)
|
||
if renewed:
|
||
continue
|
||
# 续期失败：先确认锁是否真丢（也可能只是 Redis 瞬时抖动）
|
||
if await crud.get(key=SCHEDULER_LOCK_KEY) == cls._leader_token:
|
||
continue
|
||
logger.error("调度器持有锁已丢失，本进程停机让位，等待候选接管")
|
||
cls._leader_token = None
|
||
cls._stop_scheduler_local()
|
||
else:
|
||
await asyncio.sleep(SCHEDULER_POLL_INTERVAL)
|
||
acquired, token = await crud.lock(key=SCHEDULER_LOCK_KEY, expire=SCHEDULER_LOCK_TTL)
|
||
if not acquired:
|
||
continue
|
||
cls._leader_token = token
|
||
cls._start_scheduler_local()
|
||
cls._spawn_reconcile()
|
||
logger.info("✅ 本进程接管调度器持有权，定时任务调度器已启动")
|
||
except asyncio.CancelledError:
|
||
raise
|
||
except Exception as e:
|
||
logger.error("调度器维护协程异常退出: {}", e)
|
||
|
||
@classmethod
|
||
def _spawn_reconcile(cls) -> None:
|
||
"""启动（或复用）节点任务自愈注册协程：以 task_node 表为准恢复正式调度计划。"""
|
||
if cls._reconcile_task is not None and not cls._reconcile_task.done():
|
||
return
|
||
cls._reconcile_task = asyncio.create_task(cls.reconcile_node_jobs(), name="scheduler-node-reconcile")
|
||
|
||
@classmethod
|
||
async def reconcile_node_jobs(cls) -> None:
|
||
"""节点任务自愈注册：以 task_node 表为唯一真源恢复正式调度计划。
|
||
|
||
- DB 记录（启用 + 配置了触发方式）在调度器中缺失 → 补注册（job id = 节点 id）；
|
||
- 调度器中残留的正式任务（job id 为纯数字）在 DB 已停用/删除/取消计划 → 移除；
|
||
- 已过去的一次性 date 计划不补注册（单次任务执行完成后即完成使命）；
|
||
- 手动执行产生的临时 job 不受影响；被 job 页暂停的 job 保留其暂停状态。
|
||
|
||
注意：若本进程不是调度器持有者，此处补注册的 job 会写入共享 jobstore，
|
||
需等持有者下一次 wakeup 才会真正接管（单机部署无此延迟）。
|
||
"""
|
||
from zoneinfo import ZoneInfo
|
||
|
||
from sqlalchemy import false, select
|
||
|
||
from app.core.database import async_db_session
|
||
from app.modules.task.cronjob.node.model import NodeModel
|
||
from app.modules.task.cronjob.node.service import register_node_job
|
||
|
||
try:
|
||
async with async_db_session() as session:
|
||
rows = (await session.execute(select(NodeModel).where(NodeModel.is_deleted == false()))).scalars().all()
|
||
nodes = list(rows)
|
||
except Exception as e:
|
||
logger.error(f"节点任务自愈: 读取节点列表失败: {e!s}", exc_info=True)
|
||
return
|
||
|
||
try:
|
||
existing = {job.id for job in cls.get_jobs()}
|
||
tz = ZoneInfo("Asia/Shanghai")
|
||
want_ids: set[str] = set()
|
||
for node in nodes:
|
||
if not node or node.status != 0 or not node.trigger or not (node.func or "").strip():
|
||
continue
|
||
# date 单次计划：执行时间已过的不再注册（执行完成后会被监听器移除）
|
||
if node.trigger == "date":
|
||
try:
|
||
run_at = datetime.strptime((node.trigger_args or "").strip(), "%Y-%m-%d %H:%M:%S").replace(tzinfo=tz)
|
||
if run_at <= datetime.now(tz):
|
||
continue
|
||
except (ValueError, TypeError):
|
||
continue
|
||
want_ids.add(str(node.id))
|
||
|
||
# 可观测性（t15）：把"已启用但当前不会执行"的节点与存量 job 显式打出来，
|
||
# 否则开关调整后只会表现为"任务不动了"，运维无从定位
|
||
for node in nodes:
|
||
if node.status != 0 or not node.trigger:
|
||
continue
|
||
schedulable, reason = cls.describe_schedulability(node.func)
|
||
if not schedulable:
|
||
logger.warning(f"节点任务不可调度（已启用但不会执行）: id={node.id} name={node.name!r} 原因={reason}")
|
||
cls.log_unschedulable_jobs(context="启动自愈")
|
||
|
||
for job_id in existing:
|
||
if job_id.isdigit() and job_id not in want_ids:
|
||
try:
|
||
cls.remove_job(job_id)
|
||
logger.info(f"节点任务自愈: 已移除节点 {job_id} 的残留任务")
|
||
except Exception:
|
||
pass
|
||
|
||
for node in nodes:
|
||
if str(node.id) not in want_ids or str(node.id) in existing:
|
||
continue
|
||
try:
|
||
register_node_job(node)
|
||
logger.info(f"节点任务自愈: 已恢复节点 {node.id} 的定时任务")
|
||
except Exception as e:
|
||
logger.error(f"节点任务自愈: 恢复节点 {node.id} 失败: {e!s}", exc_info=True)
|
||
except Exception as e:
|
||
logger.error(f"节点任务自愈执行失败: {e!s}", exc_info=True)
|
||
|
||
@classmethod
|
||
def _start_scheduler_local(cls) -> None:
|
||
"""本地启动调度器（幂等）。"""
|
||
if scheduler.running:
|
||
return
|
||
scheduler.start()
|
||
scheduler.add_listener(cls._dispatch_job_event, EVENT_ALL)
|
||
scheduler.resume()
|
||
|
||
@classmethod
|
||
def _stop_scheduler_local(cls, wait: bool = False) -> None:
|
||
"""本地停止调度器（幂等）。"""
|
||
if scheduler.running:
|
||
scheduler.shutdown(wait=wait)
|
||
|
||
@classmethod
|
||
def _spawn_maintain(cls) -> None:
|
||
"""启动（或替换）后台维护协程。"""
|
||
if cls._maintain_task is not None and not cls._maintain_task.done():
|
||
return
|
||
cls._maintain_task = asyncio.create_task(cls._maintain_loop(), name="scheduler-maintain")
|
||
|
||
@classmethod
|
||
async def start(cls, paused: bool = False) -> bool:
|
||
"""确保调度器运行（页面「启动调度器」入口）：持有锁则本地启动，否则转候选。
|
||
|
||
返回:
|
||
- bool: 调度器是否已由本进程（或其它持有进程）恢复运行。
|
||
"""
|
||
if scheduler.running:
|
||
return True
|
||
await cls._ensure_scheduler_running()
|
||
return scheduler.running or cls._redis is not None
|
||
|
||
@classmethod
|
||
async def release_lock(cls) -> None:
|
||
"""优雅退出时主动释放调度锁，消除退出到锁 TTL 过期之间的争抢窗口。
|
||
|
||
仅当本进程持有锁时操作；底层 Lua 校验 token 原子删除，不会误删其它进程的锁。
|
||
崩溃（SIGKILL）场景走不到这里，仍由锁 TTL 兜底自动过期。
|
||
"""
|
||
if cls._redis is None or not cls._leader_token:
|
||
return
|
||
from app.core.redis_crud import RedisCURD
|
||
|
||
released = await RedisCURD(cls._redis).unlock(SCHEDULER_LOCK_KEY, cls._leader_token)
|
||
cls._leader_token = None
|
||
if released:
|
||
logger.info("🔑 调度器持有锁已释放")
|
||
else:
|
||
logger.warning("调度器持有锁已易主或过期，跳过释放")
|
||
|
||
@classmethod
|
||
async def shutdown(cls, wait: bool = False) -> None:
|
||
"""停止本进程调度器并交还持有权；多 worker 下其它候选进程会自动接管。
|
||
|
||
顺序：先停本地调度器再释放锁，避免「锁已让出而本进程调度器仍在运行」
|
||
的双跑窗口；崩溃（SIGKILL）走不到这里，仍由锁 TTL 兜底自动过期。
|
||
APScheduler 的暂停/停止是进程内状态，跨 worker 的手动编排不在 SDK 职责内。
|
||
"""
|
||
if cls._maintain_task is not None and not cls._maintain_task.done():
|
||
cls._maintain_task.cancel()
|
||
cls._maintain_task = None
|
||
if cls._reconcile_task is not None and not cls._reconcile_task.done():
|
||
cls._reconcile_task.cancel()
|
||
cls._reconcile_task = None
|
||
cls._stop_scheduler_local(wait=wait)
|
||
await cls.release_lock()
|
||
|
||
@classmethod
|
||
def _get_trigger_type(cls, job_id: str) -> str:
|
||
"""获取任务的触发类型"""
|
||
job = cls.get_job(job_id=job_id)
|
||
if not job:
|
||
return "manual"
|
||
trigger = job.trigger
|
||
if isinstance(trigger, CronTrigger):
|
||
return "cron"
|
||
if isinstance(trigger, IntervalTrigger):
|
||
return "interval"
|
||
if isinstance(trigger, DateTrigger):
|
||
if trigger.run_date:
|
||
now = datetime.now(trigger.run_date.tzinfo)
|
||
diff = abs((trigger.run_date - now).total_seconds())
|
||
if diff < 60:
|
||
return "manual"
|
||
return "date"
|
||
return "manual"
|
||
|
||
@staticmethod
|
||
def _record_job_log(record: dict[str, Any]) -> None:
|
||
"""任务执行结果落库（成功/失败均可写；APScheduler 事件线程同步执行，独立短连接）。"""
|
||
from app.modules.task.cronjob.job.model import JobModel # 延迟导入：core 导入期不依赖业务层（守卫不变式 3）
|
||
|
||
with Session(engine) as session:
|
||
job_log = JobModel(**record)
|
||
session.add(job_log)
|
||
session.commit()
|
||
logger.info(f"执行日志已记录: job_id={record['job_id']}, id={job_log.id}")
|
||
|
||
@staticmethod
|
||
def _is_temp_job_id(job_id: str) -> bool:
|
||
"""是否为手动执行的一次性临时 job（node 手动执行 / job 卡片立即执行）。"""
|
||
return ":manual:" in job_id or "_run_now_" in job_id
|
||
|
||
@staticmethod
|
||
def _normalize_job_id(job_id: str) -> str:
|
||
"""临时 job 归一为所属节点 id，保证执行日志可按节点聚合。"""
|
||
if ":manual:" in job_id:
|
||
return job_id.split(":manual:")[0]
|
||
if "_run_now_" in job_id:
|
||
return job_id.split("_run_now_")[0]
|
||
return job_id
|
||
|
||
@classmethod
|
||
def _dispatch_job_event(cls, event: JobEvent) -> None:
|
||
"""APScheduler 事件统一处理（注册为 EVENT_ALL 回调），执行成功/失败均落库。"""
|
||
job_id = str(event.job_id) if hasattr(event, "job_id") else None
|
||
if not job_id:
|
||
return
|
||
|
||
if event.code == EVENT_JOB_EXECUTED:
|
||
logger.info(f"任务 {job_id} 执行成功")
|
||
cls._finish_job_record(job_id=job_id, status=JOB_STATUS_SUCCESS, detail=getattr(event, "retval", None))
|
||
elif event.code == EVENT_JOB_ERROR:
|
||
exception = getattr(event, "exception", None)
|
||
logger.error(f"任务 {job_id} 执行失败: {exception!s}")
|
||
cls._finish_job_record(job_id=job_id, status=JOB_STATUS_FAILED, detail=exception)
|
||
elif event.code == EVENT_JOB_MISSED:
|
||
logger.warning(f"任务 {job_id} 错过执行时间")
|
||
elif event.code == EVENT_JOB_SUBMITTED:
|
||
logger.info(f"任务 {job_id} 已提交执行")
|
||
elif event.code == EVENT_JOB_REMOVED:
|
||
logger.info(f"任务 {job_id} 已移除")
|
||
elif event.code == EVENT_JOB_ADDED:
|
||
logger.info(f"任务 {job_id} 已添加")
|
||
elif event.code == EVENT_ALL_JOBS_REMOVED:
|
||
logger.info("所有任务已从调度器中移除")
|
||
|
||
@classmethod
|
||
def _finish_job_record(cls, *, job_id: str, status: int, detail: Any) -> None:
|
||
"""任务一次执行结束的统一收尾：写执行日志；清理一次性/过期任务。"""
|
||
job = None
|
||
try:
|
||
job = SchedulerUtil.get_job(job_id=job_id)
|
||
except Exception:
|
||
pass
|
||
is_temp = cls._is_temp_job_id(job_id)
|
||
try:
|
||
record = {
|
||
"job_id": cls._normalize_job_id(job_id),
|
||
"job_name": job.name if job else None,
|
||
"trigger_type": "manual" if is_temp else (SchedulerUtil._get_trigger_type(job_id) if job else "manual"),
|
||
"status": status,
|
||
"error": _humanize_job_error(detail) if status == JOB_STATUS_FAILED else None,
|
||
"result": None if status == JOB_STATUS_FAILED else (str(detail)[:4000] if detail is not None else None),
|
||
"next_run_time": str(getattr(job, "next_run_time", None)) if getattr(job, "next_run_time", None) else None,
|
||
"job_state": SchedulerUtil._get_job_state(job) if job else None,
|
||
}
|
||
cls._record_job_log(record)
|
||
except Exception as e:
|
||
logger.error(f"记录执行日志出错: job_id={job_id}, error={e}", exc_info=True)
|
||
|
||
# 手动一次性任务：执行结束即从调度器移除，避免 jobstore 堆积
|
||
if is_temp:
|
||
try:
|
||
SchedulerUtil.remove_job(job_id=job_id)
|
||
except Exception:
|
||
pass
|
||
return
|
||
# 一次性 date 计划执行完成（无下次运行时间）后移除
|
||
if job and getattr(job, "next_run_time", None) is None and isinstance(getattr(job, "trigger", None), DateTrigger):
|
||
try:
|
||
SchedulerUtil.remove_job(job_id=job_id)
|
||
logger.info(f"一次性 date 任务 {job_id} 已完成，已从调度器移除")
|
||
except Exception:
|
||
pass
|
||
|
||
@classmethod
|
||
def pause(cls) -> None:
|
||
scheduler.pause()
|
||
|
||
@classmethod
|
||
def resume(cls) -> None:
|
||
scheduler.resume()
|
||
|
||
@classmethod
|
||
def is_running(cls) -> bool:
|
||
return scheduler.running
|
||
|
||
@classmethod
|
||
def is_scheduler_ready(cls) -> bool:
|
||
"""调度系统整体是否就绪：本进程运行中，或分布式锁体系可用（持有/候选均算）。
|
||
|
||
候选进程虽不直接执行调度，但持有者存在（或崩溃后由本进程自动接管），
|
||
调度能力由集群保证，不应在启动面板显示为失败。
|
||
"""
|
||
return scheduler.running or cls._redis is not None
|
||
|
||
@classmethod
|
||
def get_scheduler_state(cls) -> int:
|
||
return scheduler.state
|
||
|
||
@classmethod
|
||
def get_job(cls, job_id: str | int, jobstore: str | None = None) -> Job | None:
|
||
return scheduler.get_job(str(job_id), jobstore)
|
||
|
||
@classmethod
|
||
def get_jobs(cls, jobstore: str | None = None) -> list[Job]:
|
||
return scheduler.get_jobs(jobstore)
|
||
|
||
@classmethod
|
||
def remove_job(cls, job_id: str | int, jobstore: str | None = None) -> None:
|
||
scheduler.remove_job(str(job_id), jobstore)
|
||
|
||
@classmethod
|
||
def clear_jobs(cls) -> None:
|
||
scheduler.remove_all_jobs()
|
||
|
||
@classmethod
|
||
def print_jobs(cls, jobstore: str | None = None) -> str:
|
||
import io
|
||
|
||
output = io.StringIO()
|
||
scheduler.print_jobs(jobstore=jobstore, out=output)
|
||
return output.getvalue()
|
||
|
||
@classmethod
|
||
def pause_job(cls, job_id: str | int, jobstore: str | None = None) -> Job | None:
|
||
return scheduler.pause_job(str(job_id), jobstore)
|
||
|
||
@classmethod
|
||
def resume_job(cls, job_id: str | int, jobstore: str | None = None) -> Job | None:
|
||
return scheduler.resume_job(str(job_id), jobstore)
|
||
|
||
@classmethod
|
||
def modify_job(cls, job_id: str | int, jobstore: str | None = None, **changes) -> Job | None:
|
||
return scheduler.modify_job(str(job_id), jobstore, **changes)
|
||
|
||
@classmethod
|
||
def get_job_status(cls, job_id: str | int) -> int:
|
||
"""获取单个任务的当前状态。0=运行中 1=暂停中 2=已停止 3=未知"""
|
||
job = cls.get_job(job_id=str(job_id))
|
||
if not job:
|
||
return 3
|
||
# pending/未排程任务没有 next_run_time 属性（APScheduler 只在真正排程后设置），必须 getattr 兜底
|
||
if getattr(job, "next_run_time", None) is None:
|
||
return 1
|
||
if scheduler.state == 0:
|
||
return 2
|
||
return 0
|
||
|
||
@classmethod
|
||
def run_job_now(cls, job_id: str | int, jobstore: str | None = None) -> Job | None:
|
||
"""立即执行任务（通过临时 Job，不修改原任务 trigger）。"""
|
||
from datetime import timedelta
|
||
|
||
job = cls.get_job(job_id=job_id, jobstore=jobstore)
|
||
if not job:
|
||
return None
|
||
|
||
temp_job_id = f"{job_id}_run_now_{datetime.now().timestamp()}"
|
||
|
||
trigger = DateTrigger(run_date=datetime.now() + timedelta(seconds=0.1), timezone="Asia/Shanghai")
|
||
temp_job = scheduler.add_job(
|
||
func=job.func,
|
||
trigger=trigger,
|
||
args=job.args,
|
||
kwargs=job.kwargs,
|
||
id=temp_job_id,
|
||
name=f"{job.name}(立即执行)",
|
||
jobstore=jobstore or "default",
|
||
executor=job.executor,
|
||
max_instances=1,
|
||
)
|
||
logger.info(f"任务 {job_id} 已触发立即执行，临时任务 ID: {temp_job_id}")
|
||
return temp_job
|
||
|
||
@classmethod
|
||
def describe_schedulability(cls, code_block: str | None) -> tuple[bool, str | None]:
|
||
"""判断某个 func（代码块 / ``builtin:`` 引用）当前是否可调度。
|
||
|
||
单一事实来源：创建/更新守卫、节点列表状态字段、调度器巡检日志都调用它，
|
||
避免各处对 ``SCHEDULER_ALLOW_CODE_EXEC`` 的理解不一致。
|
||
|
||
参数:
|
||
- code_block (str | None): 节点的 func 字段。
|
||
|
||
返回:
|
||
- tuple[bool, str | None]: (是否可调度, 不可调度原因)。
|
||
"""
|
||
raw = (code_block or "").strip()
|
||
if not raw:
|
||
return False, "未配置函数或代码块（节点不会执行）"
|
||
if raw.startswith(cls.BUILTIN_PREFIX):
|
||
try:
|
||
cls.resolve_builtin_handler(raw)
|
||
except ValueError as exc:
|
||
return False, f"内置处理器不可用：{exc}"
|
||
return True, None
|
||
if not settings.SCHEDULER_ALLOW_CODE_EXEC:
|
||
return False, (
|
||
"服务端已禁用代码块执行（SCHEDULER_ALLOW_CODE_EXEC=False）；"
|
||
"请改用内置函数型任务——把 func 改为 builtin:<模块名>[.<函数名>]（不经 exec，不受该开关限制）"
|
||
)
|
||
return True, None
|
||
|
||
@classmethod
|
||
def require_schedulable(cls, code_block: str | None) -> None:
|
||
"""不可调度时抛出业务异常；可调度时直接返回。
|
||
|
||
把「取判定结果 + 抛异常」收敛到一处：``describe_schedulability`` 的 reason 是
|
||
``str | None``（可调度时为 None），调用点直接 ``CustomException(msg=reason)`` 会
|
||
传入 None 让客户端收到空文案；这里统一兜底，也避免每个调用点各写一份兜底文案。
|
||
|
||
参数:
|
||
- code_block (str | None): 节点的 func 字段。
|
||
|
||
异常:
|
||
- CustomException: 当前不可调度时抛出（含可操作原因）。
|
||
"""
|
||
schedulable, reason = cls.describe_schedulability(code_block)
|
||
if not schedulable:
|
||
raise CustomException(msg=reason or "当前任务不可调度，请检查任务配置")
|
||
|
||
@classmethod
|
||
def job_code_block(cls, job: Job | None) -> str | None:
|
||
"""从调度器 job 的 args 中取回 func（约定见 _add_job_with_trigger: args=[job_id, code_block, *args]）。
|
||
|
||
参数:
|
||
- job (Job | None): 调度器任务。
|
||
|
||
返回:
|
||
- str | None: func 原文；无法取得时返回 None。
|
||
"""
|
||
args = list(getattr(job, "args", None) or [])
|
||
if len(args) < 2 or args[1] is None:
|
||
return None
|
||
return str(args[1])
|
||
|
||
@classmethod
|
||
def iter_unschedulable_jobs(cls) -> list[tuple[str, str, str]]:
|
||
"""列出调度器中「已注册但当前不可执行」的任务：[(job_id, job_name, reason)]。
|
||
|
||
用于运维可观测性：开关调整或内置处理器改名后，存量任务不会静默失效。
|
||
|
||
返回:
|
||
- list[tuple[str, str, str]]: 不可调度任务清单。
|
||
"""
|
||
result: list[tuple[str, str, str]] = []
|
||
for job in cls.get_jobs():
|
||
schedulable, reason = cls.describe_schedulability(cls.job_code_block(job))
|
||
if not schedulable and reason:
|
||
result.append((str(job.id), str(job.name or ""), reason))
|
||
return result
|
||
|
||
@classmethod
|
||
def log_unschedulable_jobs(cls, *, context: str) -> list[tuple[str, str, str]]:
|
||
"""把「已注册但不可执行」的任务以 WARN 打出（带具体任务标识），返回同一清单。
|
||
|
||
参数:
|
||
- context (str): 调用场景（如 "启动自愈" / "调度器状态查询"），便于日志定位。
|
||
|
||
返回:
|
||
- list[tuple[str, str, str]]: 不可调度任务清单。
|
||
"""
|
||
found = cls.iter_unschedulable_jobs()
|
||
for job_id, job_name, reason in found:
|
||
logger.warning(f"[{context}] 调度器中的任务当前不可执行: id={job_id} name={job_name!r} 原因={reason}")
|
||
return found
|
||
|
||
@classmethod
|
||
def register_builtin_handler(cls, name: str, handler: Callable[..., Any]) -> None:
|
||
"""注册一个内置处理器（供无 exec 的「内置函数型任务」使用）。
|
||
|
||
参数:
|
||
- name (str): 名称（``func = "builtin:<name>"`` 时引用）。
|
||
- handler (Callable): 可调用对象。
|
||
|
||
返回:
|
||
- None
|
||
"""
|
||
cls._builtin_handlers[name] = handler
|
||
|
||
@classmethod
|
||
def resolve_builtin_handler(cls, code_block: str | None) -> Callable[..., Any] | None:
|
||
"""解析 ``builtin:<name>`` 引用；非内置引用返回 None。
|
||
|
||
参数:
|
||
- code_block (str | None): 节点的 func 字段。
|
||
|
||
返回:
|
||
- Callable | None: 内置处理器；不是内置引用时返回 None。
|
||
|
||
异常:
|
||
- ValueError: 引用形态非法、模块/函数不存在或不是可调用对象时抛出
|
||
（**绝不回退到 exec**，避免"看起来是内置任务、实际被执行成代码块"）。
|
||
"""
|
||
raw = (code_block or "").strip()
|
||
if not raw.startswith(cls.BUILTIN_PREFIX):
|
||
return None
|
||
name = raw[len(cls.BUILTIN_PREFIX) :].strip()
|
||
if not cls._BUILTIN_NAME_RE.match(name):
|
||
raise ValueError(f"内置处理器名称非法（仅允许 模块名[.函数名]，实际 {raw!r}）")
|
||
|
||
cached = cls._builtin_handlers.get(name)
|
||
if cached is not None:
|
||
return cached
|
||
|
||
module_name, _, func_name = name.partition(".")
|
||
func_name = func_name or module_name
|
||
import importlib
|
||
|
||
module_path = f"{cls.BUILTIN_HANDLERS_MODULE}.{module_name}"
|
||
try:
|
||
module = importlib.import_module(module_path)
|
||
except ModuleNotFoundError as exc:
|
||
raise ValueError(f"内置处理器模块不存在：{module_path}") from exc
|
||
handler = getattr(module, func_name, None)
|
||
if handler is None or not callable(handler):
|
||
raise ValueError(f"内置处理器不可用（{module_path}.{func_name} 不存在或不可调用）")
|
||
# 安全加固（复核 R1）：只接受「在本模块内定义」或「该模块显式声明」的可调用对象。
|
||
# 否则 handler 模块里任何一条 `from os import system` 之类的导入，都会变成
|
||
# 绕过 SCHEDULER_ALLOW_CODE_EXEC 的无限制执行入口。
|
||
declared = getattr(module, "HANDLERS", None)
|
||
if isinstance(declared, dict):
|
||
if declared.get(func_name) is not handler:
|
||
raise ValueError(f"内置处理器未在该模块的 HANDLERS 中声明：{module_path}.{func_name}")
|
||
elif getattr(handler, "__module__", None) != module.__name__:
|
||
raise ValueError(
|
||
f"内置处理器必须在本模块内定义或经 HANDLERS 声明：{module_path}.{func_name} "
|
||
f"实际来自 {getattr(handler, '__module__', '未知来源')}，已拒绝"
|
||
)
|
||
cls._builtin_handlers[name] = handler
|
||
return handler
|
||
|
||
@classmethod
|
||
def _task_wrapper(cls, job_id: str | int, code_block: str | None, *args, **kwargs):
|
||
"""任务执行包装器，执行自定义代码块（同步版本，用于 ThreadPoolExecutor）
|
||
|
||
两类任务：
|
||
- **内置函数型**（``code_block = "builtin:<模块名>[.<函数名>]"``）：走白名单注册表直接调用，
|
||
**不使用 exec**，因此不受 ``SCHEDULER_ALLOW_CODE_EXEC`` 限制；
|
||
- **代码块型**（其它内容）：``exec`` 等于给所有能创建任务的人服务器代码执行权限，**默认关闭**，
|
||
关闭时执行入口直接失败并把原因写入执行日志（而不是静默跳过）。
|
||
"""
|
||
# 内置函数型任务（builtin:<name>）：白名单解析后直接调用，不经过 exec，
|
||
# 因此即使 SCHEDULER_ALLOW_CODE_EXEC=False 也能正常调度
|
||
raw_code = (code_block or "").strip()
|
||
if raw_code.startswith(cls.BUILTIN_PREFIX):
|
||
handler = cls.resolve_builtin_handler(raw_code)
|
||
logger.info(f"任务 {job_id} 以内置处理器执行（不经 exec）: {raw_code}")
|
||
return handler(*args, **kwargs) if handler else None
|
||
|
||
if raw_code and not settings.SCHEDULER_ALLOW_CODE_EXEC:
|
||
message = (
|
||
f"任务 {job_id} 含用户提交的代码块，已拒绝执行：服务端默认禁止定时任务执行代码块。"
|
||
f"如确需该能力，请显式设置 SCHEDULER_ALLOW_CODE_EXEC=true（等同授予服务器代码执行权限，请谨慎评估）；"
|
||
f"若只需内置能力，可把 func 改为 builtin:<模块名>[.<函数名>]"
|
||
)
|
||
logger.error(message)
|
||
raise RuntimeError(message)
|
||
|
||
import types
|
||
|
||
def run_sync_handler():
|
||
if not code_block:
|
||
return None
|
||
module = types.ModuleType(f"node_task_{job_id}")
|
||
module.__dict__["__builtins__"] = __builtins__
|
||
exec(code_block, module.__dict__)
|
||
handler = module.__dict__.get("handler")
|
||
if handler and callable(handler):
|
||
return handler(*args, **kwargs)
|
||
raise ValueError("代码块必须定义 handler(*args, **kwargs) 函数")
|
||
|
||
try:
|
||
return run_sync_handler()
|
||
except Exception as e:
|
||
logger.error(f"任务 {job_id} 执行失败: {e!s}")
|
||
raise
|
||
|
||
@classmethod
|
||
def _get_job_state(cls, job) -> str | None:
|
||
"""获取任务状态（解析为可读的JSON格式）"""
|
||
import pickle
|
||
|
||
if not job:
|
||
return None
|
||
state = job.__getstate__()
|
||
|
||
def serialize_value(obj):
|
||
if obj is None:
|
||
return None
|
||
if isinstance(obj, (str, int, float, bool)):
|
||
return obj
|
||
if isinstance(obj, bytes):
|
||
try:
|
||
return serialize_value(pickle.loads(obj))
|
||
except Exception:
|
||
return obj.decode("utf-8", errors="replace")
|
||
if isinstance(obj, dict):
|
||
return {k: serialize_value(v) for k, v in obj.items()}
|
||
if isinstance(obj, (list, tuple)):
|
||
return [serialize_value(item) for item in obj]
|
||
if hasattr(obj, "__dict__"):
|
||
obj_dict = {}
|
||
for k, v in obj.__dict__.items():
|
||
if not k.startswith("_"):
|
||
obj_dict[k] = serialize_value(v)
|
||
return {"__class__": obj.__class__.__name__, **obj_dict}
|
||
try:
|
||
return str(obj)
|
||
except Exception:
|
||
return f"<{type(obj).__name__}>"
|
||
|
||
return json.dumps(serialize_value(state), ensure_ascii=False, indent=2)
|