diff --git a/backend/app/config/setting.py b/backend/app/config/setting.py index 6f6088e5..53ae4299 100755 --- a/backend/app/config/setting.py +++ b/backend/app/config/setting.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import Any, Literal from urllib.parse import quote_plus +from pydantic import model_validator from pydantic_settings import BaseSettings, SettingsConfigDict from app.common.enums import EnvironmentEnum @@ -54,7 +55,8 @@ class Settings(BaseSettings): # ================================================= # # ******************** 跨域配置 ******************** # # ================================================= # - PROD_CORS_ORIGINS: str = "" # 生产环境允许的域名列表，逗号分隔，如 "https://admin.example.com,https://www.example.com" + PROD_CORS_ORIGINS: str = "" # 生产环境允许的域名列表，逗号分隔，如 "https://admin.example.com,https://www.example.com"；生产留空=拒绝全部跨域请求 + CORS_STRICT_STARTUP: bool = False # True 时「生产未配置 PROD_CORS_ORIGINS」直接启动失败（默认只告警，保持向后兼容） ALLOW_METHODS: list[str] = ["*"] # 允许的HTTP方法 ALLOW_HEADERS: list[str] = ["*"] # 允许的请求头 ALLOW_CREDENTIALS: bool = True # 是否允许携带cookie @@ -124,7 +126,7 @@ class Settings(BaseSettings): # ================================================= # # ******************* 任务调度配置 ****************** # # ================================================= # - SCHEDULER_ALLOW_CODE_EXEC: bool = True # 是否允许定时任务执行用户提交的代码块(exec)。等同远程代码执行能力，生产环境强烈建议设为 False + SCHEDULER_ALLOW_CODE_EXEC: bool = False # 是否允许定时任务执行用户提交的代码块(exec)。等同远程代码执行能力，默认关闭，仅在显式配置为 true 时启用 # ================================================= # # ******************* 口令策略配置 ****************** # @@ -147,7 +149,9 @@ class Settings(BaseSettings): OAUTH_QQ_APP_ID: str = "" OAUTH_QQ_APP_SECRET: str = "" OAUTH_STATE_TTL: int = 600 # OAuth state 参数过期时间（秒） - OAUTH_ALLOWED_HOSTS: list[str] = ["*"] + # 生产回跳域名白名单（**不再是通配符 "*"**）：OAuth 回调域名必须命中其中之一。 + # 非生产环境通常用 localhost/容器域名登录，故默认一并放行本机与回环地址；生产可按需收敛。 + OAUTH_ALLOWED_HOSTS: list[str] = ["service.fastapiadmin.com", "*.fastapiadmin.com", "localhost", "127.0.0.1", "[::1]"] # ================================================= # # *************** 微信小程序配置（可选）************** # @@ -182,7 +186,11 @@ class Settings(BaseSettings): # ================================================= # # ******************* 安全中间件配置 ****************** # # ================================================= # - ALLOWED_HOSTS: list[str] = ["service.fastapiadmin.com", "*.fastapiadmin.com"] # 允许访问的主机名列表 + ALLOWED_HOSTS: list[str] = ["service.fastapiadmin.com", "*.fastapiadmin.com"] # 允许访问的主机名列表（生产域名） + # 容器内健康检查走 http://localhost:8001、本地直连调试都需要 Host: localhost/127.0.0.1/[::1]； + # 该放行范围由下面的开关控制——生产若不需要（例如健康检查改走 127.0.0.1 之外的内部域名）可置 False。 + ALLOW_LOCALHOST_HOSTS: bool = True + LOCALHOST_HOSTS: list[str] = ["localhost", "127.0.0.1", "[::1]"] # ALLOW_LOCALHOST_HOSTS=True 时自动并入 ALLOWED_HOSTS # 接口白名单（无需认证即可访问的接口路径，支持 * 开头表示前缀匹配） WHITE_API_LIST_PATH: list[str] = [ @@ -236,15 +244,70 @@ class Settings(BaseSettings): OPENAI_MODEL: str = "" OPENAI_BASE_URL: str = "" # API Base URL，如 https://api.minimax.chat/v1 + @model_validator(mode="after") + def _merge_localhost_hosts(self) -> "Settings": + """按开关把 localhost/回环地址并入 ALLOWED_HOSTS。 + + 健康检查与本地联调需要 ``Host: localhost``；把它们做成可配置项（而不是无条件写死）， + 生产可以用 ``ALLOW_LOCALHOST_HOSTS=False`` 收敛 Host 头校验范围。 + + 返回: + - Settings: 自身（pydantic after-validator 约定）。 + """ + if self.ALLOW_LOCALHOST_HOSTS: + for host in self.LOCALHOST_HOSTS: + if host not in self.ALLOWED_HOSTS: + self.ALLOWED_HOSTS.append(host) + return self + # ================================================= # # ******************* 动态配置 ******************* # # ================================================= # @property def ALLOW_ORIGINS(self) -> list[str]: - """根据环境动态返回 CORS 允许的域名列表。""" - if self.ENVIRONMENT == EnvironmentEnum.PROD and self.PROD_CORS_ORIGINS: - return [origin.strip() for origin in self.PROD_CORS_ORIGINS.split(",") if origin.strip()] - return ["*"] + """根据环境动态返回 CORS 允许的域名列表。 + + 安全默认（见 backend/audit-security.md S9）： + - 非生产环境：返回 ``["*"]``，保留本地联调便利； + - 生产环境：只接受**显式配置**的域名清单。未配置、或清单里只有通配符 ``"*"`` 时返回空列表 + （空列表 = 不下发任何跨域许可响应头；同源部署的 Web 端不受影响），并打 warning 提示运维配置。 + 历史行为是"未配置就回落 ``["*"]``"，叠加 ``ALLOW_CREDENTIALS=True`` 等价于允许任意站点 + 带凭据跨域访问。 + """ + if self.ENVIRONMENT != EnvironmentEnum.PROD: + return ["*"] + + origins = [origin.strip() for origin in (self.PROD_CORS_ORIGINS or "").split(",") if origin.strip()] + if "*" in origins: + origins = [origin for origin in origins if origin != "*"] + self._warn_insecure_cors( + "生产环境 CORS 不允许通配符 '*'（与 ALLOW_CREDENTIALS=True 组合会放行任意站点带凭据跨域），已忽略该值；请显式配置 PROD_CORS_ORIGINS", + ) + if not origins: + message = "生产环境未显式配置 PROD_CORS_ORIGINS，已按安全默认拒绝全部跨域请求（同源部署不受影响）；如需跨域请在 .env.prod 中配置 PROD_CORS_ORIGINS" + if self.CORS_STRICT_STARTUP: + # 严格模式：宁可启动失败，也不要带着"跨域被全量拒绝"的隐性配置上线 + raise RuntimeError(f"CORS_STRICT_STARTUP=True 且 {message}") + self._warn_insecure_cors(message) + return origins + + @staticmethod + def _warn_insecure_cors(message: str) -> None: + """输出安全告警（日志不可用时静默，绝不影响配置读取）。 + + 参数: + - message (str): 告警内容。 + + 返回: + - None + """ + # 延迟导入：app.core.logger 会反向 import 本模块，模块级导入会形成循环依赖 + try: + from app.core.logger import logger + + logger.warning(message) + except Exception: # noqa: BLE001 - 告警失败不得影响配置读取 + pass # ================================================= # @property diff --git a/backend/app/core/ap_scheduler.py b/backend/app/core/ap_scheduler.py index 4ec95fe4..24de7510 100644 --- a/backend/app/core/ap_scheduler.py +++ b/backend/app/core/ap_scheduler.py @@ -1,5 +1,7 @@ import asyncio import json +import re +from collections.abc import Callable from datetime import datetime from typing import Any @@ -94,6 +96,16 @@ class SchedulerUtil: """定时任务 SDK — 封装 APScheduler 核心操作与任务执行记录落库。""" redis_instance: Redis | None = None + # ── 内置处理器白名单（「脚本类」与「内置函数类」任务的分界）──────────────── + # 节点任务 func 形如 "builtin:<模块名>[.<函数名>]"（模块限定在 handlers 包内）时， + # 由注册表解析后**直接调用**，不经过 exec —— 因此不受 SCHEDULER_ALLOW_CODE_EXEC 限制。 + # 这样该开关只关闭"执行用户提交代码块"的能力，而不是整个调度特性。 + BUILTIN_PREFIX = "builtin:" + BUILTIN_HANDLERS_MODULE = "app.modules.task.cronjob.node.handlers" + # 名称白名单：仅允许「模块名」或「模块名.函数名」，禁止点号穿越 / 括号 / 特殊字符 + _BUILTIN_NAME_RE = re.compile(r"^[a-z_][a-z0-9_]*(\.[a-z_][a-z0-9_]*)?$") + _builtin_handlers: dict[str, Callable[..., Any]] = {} + # 多 worker 选主状态：_redis 为空视为单机直跑；_leader_token 非空表示本进程持有调度锁 _redis: Redis | None = None _leader_token: str | None = None @@ -244,6 +256,16 @@ class SchedulerUtil: continue want_ids.add(str(node.id)) + # 可观测性（t15）：把"已启用但当前不会执行"的节点与存量 job 显式打出来， + # 否则开关调整后只会表现为"任务不动了"，运维无从定位 + for node in nodes: + if node.status != 0 or not node.trigger: + continue + schedulable, reason = cls.describe_schedulability(node.func) + if not schedulable: + logger.warning(f"节点任务不可调度（已启用但不会执行）: id={node.id} name={node.name!r} 原因={reason}") + cls.log_unschedulable_jobs(context="启动自愈") + for job_id in existing: if job_id.isdigit() and job_id not in want_ids: try: @@ -419,7 +441,7 @@ class SchedulerUtil: "status": status, "error": _humanize_job_error(detail) if status == JOB_STATUS_FAILED else None, "result": None if status == JOB_STATUS_FAILED else (str(detail)[:4000] if detail is not None else None), - "next_run_time": str(job.next_run_time) if job and job.next_run_time else None, + "next_run_time": str(getattr(job, "next_run_time", None)) if getattr(job, "next_run_time", None) else None, "job_state": SchedulerUtil._get_job_state(job) if job else None, } cls._record_job_log(record) @@ -434,7 +456,7 @@ class SchedulerUtil: pass return # 一次性 date 计划执行完成（无下次运行时间）后移除 - if job and job.next_run_time is None and isinstance(getattr(job, "trigger", None), DateTrigger): + if job and getattr(job, "next_run_time", None) is None and isinstance(getattr(job, "trigger", None), DateTrigger): try: SchedulerUtil.remove_job(job_id=job_id) logger.info(f"一次性 date 任务 {job_id} 已完成，已从调度器移除") @@ -508,7 +530,8 @@ class SchedulerUtil: job = cls.get_job(job_id=str(job_id)) if not job: return 3 - if job.next_run_time is None: + # pending/未排程任务没有 next_run_time 属性（APScheduler 只在真正排程后设置），必须 getattr 兜底 + if getattr(job, "next_run_time", None) is None: return 1 if scheduler.state == 0: return 2 @@ -540,16 +563,170 @@ class SchedulerUtil: logger.info(f"任务 {job_id} 已触发立即执行，临时任务 ID: {temp_job_id}") return temp_job + @classmethod + def describe_schedulability(cls, code_block: str | None) -> tuple[bool, str | None]: + """判断某个 func（代码块 / ``builtin:`` 引用）当前是否可调度。 + + 单一事实来源：创建/更新守卫、节点列表状态字段、调度器巡检日志都调用它， + 避免各处对 ``SCHEDULER_ALLOW_CODE_EXEC`` 的理解不一致。 + + 参数: + - code_block (str | None): 节点的 func 字段。 + + 返回: + - tuple[bool, str | None]: (是否可调度, 不可调度原因)。 + """ + raw = (code_block or "").strip() + if not raw: + return False, "未配置函数或代码块（节点不会执行）" + if raw.startswith(cls.BUILTIN_PREFIX): + try: + cls.resolve_builtin_handler(raw) + except ValueError as exc: + return False, f"内置处理器不可用：{exc}" + return True, None + if not settings.SCHEDULER_ALLOW_CODE_EXEC: + return False, ( + "服务端已禁用代码块执行（SCHEDULER_ALLOW_CODE_EXEC=False）；" + "请改用内置函数型任务——把 func 改为 builtin:<模块名>[.<函数名>]（不经 exec，不受该开关限制）" + ) + return True, None + + @classmethod + def job_code_block(cls, job: Job | None) -> str | None: + """从调度器 job 的 args 中取回 func（约定见 _add_job_with_trigger: args=[job_id, code_block, *args]）。 + + 参数: + - job (Job | None): 调度器任务。 + + 返回: + - str | None: func 原文；无法取得时返回 None。 + """ + args = list(getattr(job, "args", None) or []) + if len(args) < 2 or args[1] is None: + return None + return str(args[1]) + + @classmethod + def iter_unschedulable_jobs(cls) -> list[tuple[str, str, str]]: + """列出调度器中「已注册但当前不可执行」的任务：[(job_id, job_name, reason)]。 + + 用于运维可观测性：开关调整或内置处理器改名后，存量任务不会静默失效。 + + 返回: + - list[tuple[str, str, str]]: 不可调度任务清单。 + """ + result: list[tuple[str, str, str]] = [] + for job in cls.get_jobs(): + schedulable, reason = cls.describe_schedulability(cls.job_code_block(job)) + if not schedulable and reason: + result.append((str(job.id), str(job.name or ""), reason)) + return result + + @classmethod + def log_unschedulable_jobs(cls, *, context: str) -> list[tuple[str, str, str]]: + """把「已注册但不可执行」的任务以 WARN 打出（带具体任务标识），返回同一清单。 + + 参数: + - context (str): 调用场景（如 "启动自愈" / "调度器状态查询"），便于日志定位。 + + 返回: + - list[tuple[str, str, str]]: 不可调度任务清单。 + """ + found = cls.iter_unschedulable_jobs() + for job_id, job_name, reason in found: + logger.warning(f"[{context}] 调度器中的任务当前不可执行: id={job_id} name={job_name!r} 原因={reason}") + return found + + @classmethod + def register_builtin_handler(cls, name: str, handler: Callable[..., Any]) -> None: + """注册一个内置处理器（供无 exec 的「内置函数型任务」使用）。 + + 参数: + - name (str): 名称（``func = "builtin:"`` 时引用）。 + - handler (Callable): 可调用对象。 + + 返回: + - None + """ + cls._builtin_handlers[name] = handler + + @classmethod + def resolve_builtin_handler(cls, code_block: str | None) -> Callable[..., Any] | None: + """解析 ``builtin:`` 引用；非内置引用返回 None。 + + 参数: + - code_block (str | None): 节点的 func 字段。 + + 返回: + - Callable | None: 内置处理器；不是内置引用时返回 None。 + + 异常: + - ValueError: 引用形态非法、模块/函数不存在或不是可调用对象时抛出 + （**绝不回退到 exec**，避免"看起来是内置任务、实际被执行成代码块"）。 + """ + raw = (code_block or "").strip() + if not raw.startswith(cls.BUILTIN_PREFIX): + return None + name = raw[len(cls.BUILTIN_PREFIX) :].strip() + if not cls._BUILTIN_NAME_RE.match(name): + raise ValueError(f"内置处理器名称非法（仅允许 模块名[.函数名]，实际 {raw!r}）") + + cached = cls._builtin_handlers.get(name) + if cached is not None: + return cached + + module_name, _, func_name = name.partition(".") + func_name = func_name or module_name + import importlib + + module_path = f"{cls.BUILTIN_HANDLERS_MODULE}.{module_name}" + try: + module = importlib.import_module(module_path) + except ModuleNotFoundError as exc: + raise ValueError(f"内置处理器模块不存在：{module_path}") from exc + handler = getattr(module, func_name, None) + if handler is None or not callable(handler): + raise ValueError(f"内置处理器不可用（{module_path}.{func_name} 不存在或不可调用）") + # 安全加固（复核 R1）：只接受「在本模块内定义」或「该模块显式声明」的可调用对象。 + # 否则 handler 模块里任何一条 `from os import system` 之类的导入，都会变成 + # 绕过 SCHEDULER_ALLOW_CODE_EXEC 的无限制执行入口。 + declared = getattr(module, "HANDLERS", None) + if isinstance(declared, dict): + if declared.get(func_name) is not handler: + raise ValueError(f"内置处理器未在该模块的 HANDLERS 中声明：{module_path}.{func_name}") + elif getattr(handler, "__module__", None) != module.__name__: + raise ValueError( + f"内置处理器必须在本模块内定义或经 HANDLERS 声明：{module_path}.{func_name} " + f"实际来自 {getattr(handler, '__module__', '未知来源')}，已拒绝" + ) + cls._builtin_handlers[name] = handler + return handler + @classmethod def _task_wrapper(cls, job_id: str | int, code_block: str | None, *args, **kwargs): """任务执行包装器，执行自定义代码块（同步版本，用于 ThreadPoolExecutor） - 安全提示：code_block 来自页面提交，exec 等同给所有能创建任务的人 - 服务器代码执行权限。生产环境应设置 SCHEDULER_ALLOW_CODE_EXEC=False - 关闭该能力，仅保留内置函数型任务。 + 两类任务： + - **内置函数型**（``code_block = "builtin:<模块名>[.<函数名>]"``）：走白名单注册表直接调用， + **不使用 exec**，因此不受 ``SCHEDULER_ALLOW_CODE_EXEC`` 限制； + - **代码块型**（其它内容）：``exec`` 等于给所有能创建任务的人服务器代码执行权限，**默认关闭**， + 关闭时执行入口直接失败并把原因写入执行日志（而不是静默跳过）。 """ - if code_block and code_block.strip() and not settings.SCHEDULER_ALLOW_CODE_EXEC: - message = f"任务 {job_id} 含用户提交代码块，已因 SCHEDULER_ALLOW_CODE_EXEC=False 拒绝执行" + # 内置函数型任务（builtin:）：白名单解析后直接调用，不经过 exec， + # 因此即使 SCHEDULER_ALLOW_CODE_EXEC=False 也能正常调度 + raw_code = (code_block or "").strip() + if raw_code.startswith(cls.BUILTIN_PREFIX): + handler = cls.resolve_builtin_handler(raw_code) + logger.info(f"任务 {job_id} 以内置处理器执行（不经 exec）: {raw_code}") + return handler(*args, **kwargs) if handler else None + + if raw_code and not settings.SCHEDULER_ALLOW_CODE_EXEC: + message = ( + f"任务 {job_id} 含用户提交的代码块，已拒绝执行：服务端默认禁止定时任务执行代码块。" + f"如确需该能力，请显式设置 SCHEDULER_ALLOW_CODE_EXEC=true（等同授予服务器代码执行权限，请谨慎评估）；" + f"若只需内置能力，可把 func 改为 builtin:<模块名>[.<函数名>]" + ) logger.error(message) raise RuntimeError(message) diff --git a/backend/app/core/base_crud.py b/backend/app/core/base_crud.py index 2055458f..8dee7c22 100644 --- a/backend/app/core/base_crud.py +++ b/backend/app/core/base_crud.py @@ -78,7 +78,7 @@ class CRUDBase[ModelType: ModelMixin, CreateSchemaType, UpdateSchemaType]: result: Result = await self.db.execute(sql) return result.scalars().first() except Exception as e: - raise CustomException(msg=f"获取查询失败: {e!s}") from e + raise CustomException(msg=f"获取查询失败: {e!s}", status_code=500) from e async def get_or_404( self, @@ -89,12 +89,15 @@ class CRUDBase[ModelType: ModelMixin, CreateSchemaType, UpdateSchemaType]: include_deleted: bool = False, **kwargs, ) -> ModelType | OutSchemaType: - """查不到直接抛异常。支持 id 快捷入参，也支持 **kwargs 传多个条件。""" + """查不到直接抛异常。支持 id 快捷入参，也支持 **kwargs 传多个条件。 + + 「资源不存在」属客户端可修正的结果：按 404 返回（业务 code 保持默认，前端仍展示 msg）。 + """ if id is not None: kwargs["id"] = id obj = await self.get(preload=preload, include_deleted=include_deleted, **kwargs) if not obj: - raise CustomException(msg=msg) + raise CustomException(msg=msg, status_code=404) return out_schema.model_validate(obj) if out_schema else obj async def exists(self, include_deleted: bool = False, **kwargs) -> bool: @@ -109,7 +112,7 @@ class CRUDBase[ModelType: ModelMixin, CreateSchemaType, UpdateSchemaType]: result: Result = await self.db.execute(count_sql) return result.scalar() or 0 except Exception as e: - raise CustomException(msg=f"统计失败: {e!s}") from e + raise CustomException(msg=f"统计失败: {e!s}", status_code=500) from e async def get_list( self, @@ -131,7 +134,7 @@ class CRUDBase[ModelType: ModelMixin, CreateSchemaType, UpdateSchemaType]: result: Result = await self.db.execute(sql) return result.scalars().all() except Exception as e: - raise CustomException(msg=f"列表查询失败: {e!s}") from e + raise CustomException(msg=f"列表查询失败: {e!s}", status_code=500) from e async def page( self, @@ -179,7 +182,7 @@ class CRUDBase[ModelType: ModelMixin, CreateSchemaType, UpdateSchemaType]: items=items, ) except Exception as e: - raise CustomException(msg=f"分页查询失败: {e!s}") from e + raise CustomException(msg=f"分页查询失败: {e!s}", status_code=500) from e # ── 写入 ────────────────────────────────────────────────────────── @@ -215,7 +218,7 @@ class CRUDBase[ModelType: ModelMixin, CreateSchemaType, UpdateSchemaType]: return obj except Exception as e: - raise CustomException(msg=f"创建失败: {e!s}") from e + raise CustomException(msg=f"创建失败: {e!s}", status_code=500) from e async def update(self, id: int, data: UpdateSchemaType | dict[str, Any]) -> ModelType: """更新记录。用 exclude_unset / exclude_none 准确表达前端意图。""" @@ -253,7 +256,7 @@ class CRUDBase[ModelType: ModelMixin, CreateSchemaType, UpdateSchemaType]: except CustomException: raise except Exception as e: - raise CustomException(msg=f"更新失败: {e!s}") from e + raise CustomException(msg=f"更新失败: {e!s}", status_code=500) from e async def delete(self, ids: list[int]) -> None: """软删除优先，无软删除则物理删除。""" @@ -269,7 +272,7 @@ class CRUDBase[ModelType: ModelMixin, CreateSchemaType, UpdateSchemaType]: await self.db.execute(sql) await self.db.flush() except Exception as e: - raise CustomException(msg=f"删除失败: {e!s}") from e + raise CustomException(msg=f"删除失败: {e!s}", status_code=500) from e async def clear(self) -> None: """清空整表。软删除模式下相当于"回收站清空"，只清理已删标记的记录。""" @@ -283,7 +286,7 @@ class CRUDBase[ModelType: ModelMixin, CreateSchemaType, UpdateSchemaType]: await self.db.execute(sql) await self.db.flush() except Exception as e: - raise CustomException(msg=f"清空失败: {e!s}") from e + raise CustomException(msg=f"清空失败: {e!s}", status_code=500) from e async def set(self, ids: list[int], include_deleted: bool = False, **kwargs) -> None: """批量更新。软删除模式下默认跳过已删除的记录。""" @@ -296,7 +299,7 @@ class CRUDBase[ModelType: ModelMixin, CreateSchemaType, UpdateSchemaType]: await self.db.execute(sql) await self.db.flush() except Exception as e: - raise CustomException(msg=f"批量更新失败: {e!s}") from e + raise CustomException(msg=f"批量更新失败: {e!s}", status_code=500) from e # ── 条件与排序 ──────────────────────────────────────────────────── diff --git a/backend/app/core/exceptions.py b/backend/app/core/exceptions.py index 6e38c106..ae611fe3 100644 --- a/backend/app/core/exceptions.py +++ b/backend/app/core/exceptions.py @@ -38,17 +38,72 @@ def require_superadmin(func): return wrapper +# starlette 新版本把 HTTP_422_UNPROCESSABLE_ENTITY 标为弃用，这里取新常量并兼容旧版本， +# 避免模块导入期就打 DeprecationWarning +_HTTP_422: int = getattr(status, "HTTP_422_UNPROCESSABLE_CONTENT", 422) + +# 业务错误码 → HTTP 状态码。只映射语义明确的码；其余业务异常统一按 _DEFAULT_BUSINESS_STATUS 返回。 +# 说明：历史上 CustomException 的 status_code 默认 500，导致「验证码过期」「参数校验失败」这类 +# 客户端错误也返回 500 —— 前端无法区分「我错了」与「服务器坏了」，监控/告警也被污染。 +_CODE_TO_HTTP_STATUS: dict[int, int] = { + RET.BAD_REQUEST.code: status.HTTP_400_BAD_REQUEST, + RET.UNAUTHORIZED.code: status.HTTP_401_UNAUTHORIZED, + RET.FORBIDDEN.code: status.HTTP_403_FORBIDDEN, + RET.NOT_FOUND.code: status.HTTP_404_NOT_FOUND, + RET.CONFLICT.code: status.HTTP_409_CONFLICT, + RET.UNPROCESSABLE_ENTITY.code: _HTTP_422, + RET.TOO_MANY_REQUESTS.code: status.HTTP_429_TOO_MANY_REQUESTS, + RET.SERVICE_UNAVAILABLE.code: status.HTTP_503_SERVICE_UNAVAILABLE, + RET.DATAEXIST.code: status.HTTP_409_CONFLICT, + RET.PARAMERR.code: status.HTTP_400_BAD_REQUEST, + RET.TIMEOUT.code: status.HTTP_504_GATEWAY_TIMEOUT, + RET.RATE_LIMIT_EXCEEDED.code: status.HTTP_429_TOO_MANY_REQUESTS, + # 认证类业务码（对应前端 ResultEnum） + RET.INVALID_TOKEN.code: status.HTTP_401_UNAUTHORIZED, + RET.EXPIRED_TOKEN.code: status.HTTP_401_UNAUTHORIZED, + RET.INVALID_CREDENTIALS.code: status.HTTP_401_UNAUTHORIZED, + RET.TOKEN_EXPIRED.code: status.HTTP_401_UNAUTHORIZED, + RET.NO_PERMISSION.code: status.HTTP_403_FORBIDDEN, + # 显式声明为服务端错误的码（供内部包装器使用，如 CRUD「xx失败」兜底） + RET.SERVERERR.code: status.HTTP_500_INTERNAL_SERVER_ERROR, +} + +# 未映射业务码的默认状态：业务异常按「客户端请求错误」处理（而不是 500） +_DEFAULT_BUSINESS_STATUS: int = status.HTTP_400_BAD_REQUEST + + +def resolve_http_status(code: int | None) -> int: + """把业务错误码解析为 HTTP 状态码。 + + 参数: + - code (int | None): 业务错误码。 + + 返回: + - int: 对应的 HTTP 状态码；未映射的码返回 _DEFAULT_BUSINESS_STATUS。 + """ + if code is None: + return _DEFAULT_BUSINESS_STATUS + return _CODE_TO_HTTP_STATUS.get(int(code), _DEFAULT_BUSINESS_STATUS) + + class CustomException(Exception): + """业务异常。 + + ``status_code`` 未显式传入时按 ``code`` 推导（见 :func:`resolve_http_status`）， + 默认 400（客户端可修正）而非 500；只有确属服务端故障的场景才应显式传 500， + 或使用 ``code=RET.SERVERERR.code``。 + """ + def __init__( self, msg: str = RET.EXCEPTION.msg, code: int = RET.EXCEPTION.code, - status_code: int = status.HTTP_500_INTERNAL_SERVER_ERROR, + status_code: int | None = None, data: Any | None = None, success: bool = False, ) -> None: super().__init__(msg) - self.status_code = status_code + self.status_code = status_code if status_code is not None else resolve_http_status(code) self.code = code self.msg = msg self.data = data @@ -61,11 +116,15 @@ class CustomException(Exception): def handle_exception(app: FastAPI) -> None: @app.exception_handler(CustomException) async def custom_exception_handler(request: Request, exc: CustomException) -> JSONResponse: - logger.error( - "[自定义异常] {} {} | code={} | msg={} | data={}", + # 4xx 是「客户端可修正」的业务结果，5xx 才是服务端故障：分级记录， + # 避免每个校验失败都打 ERROR 把真正的故障信号淹没。 + log = logger.error if exc.status_code >= status.HTTP_500_INTERNAL_SERVER_ERROR else logger.warning + log( + "[自定义异常] {} {} | code={} | status={} | msg={} | data={}", request.method, request.url.path, exc.code, + exc.status_code, exc.msg, exc.data, ) @@ -96,7 +155,7 @@ def handle_exception(app: FastAPI) -> None: request.url.path, errors, ) - return ErrorResponse(msg=str(msg), status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, data=errors) + return ErrorResponse(msg=str(msg), status_code=_HTTP_422, data=errors) @app.exception_handler(ResponseValidationError) async def response_validation_handler(request: Request, exc: ResponseValidationError) -> JSONResponse: diff --git a/backend/app/modules/ai/chat/crud.py b/backend/app/modules/ai/chat/crud.py index 87dc900f..9bb2d577 100644 --- a/backend/app/modules/ai/chat/crud.py +++ b/backend/app/modules/ai/chat/crud.py @@ -42,7 +42,8 @@ def _get_agno_db() -> Any: elif db_type == "sqlite": _db_instance = AsyncSqliteDb(db_file=db_uri.replace("sqlite+aiosqlite:///", "")) else: - raise CustomException(msg=f"不支持的数据库类型: {db_type}") + # settings.DATABASE_TYPE 是服务端配置：取值不在支持范围属部署配置错误，按 500 返回 + raise CustomException(msg=f"不支持的数据库类型: {db_type}", status_code=500) return _db_instance @@ -141,7 +142,7 @@ class ChatSessionCRUD: ) except Exception as e: logger.error(f"获取会话列表失败: {e}") - raise CustomException(msg="获取会话列表失败") from e + raise CustomException(msg="获取会话列表失败", status_code=500) from e rows: list[dict[str, Any]] = [] total = 0 @@ -173,9 +174,9 @@ class ChatSessionCRUD: result = await self.db.upsert_session(session=session) except Exception as e: logger.exception(f"创建会话失败: {e}") - raise CustomException(msg="创建会话失败") from e + raise CustomException(msg="创建会话失败", status_code=500) from e if result is None: - raise CustomException(msg="创建会话失败") + raise CustomException(msg="创建会话失败", status_code=500) return result async def rename(self, session_id: str, data: ChatSessionUpdateSchema) -> None: @@ -189,7 +190,7 @@ class ChatSessionCRUD: ) except Exception as e: logger.error(f"更新会话失败: {e}") - raise CustomException(msg="更新会话失败") from e + raise CustomException(msg="更新会话失败", status_code=500) from e if result is None: raise CustomException(msg="会话不存在", code=10404, status_code=404) @@ -200,4 +201,5 @@ class ChatSessionCRUD: await self.db.delete_session(session_id=session_id, user_id=self.user_id) except Exception as e: logger.error(f"删除会话失败: {session_id} - {e}") - raise CustomException(msg=f"删除会话失败: {session_id}") from e + # agno/SQLite 侧删除失败属内部故障，按 500 返回（session_ids 为客户端参数，但失败原因在服务端） + raise CustomException(msg=f"删除会话失败: {session_id}", status_code=500) from e diff --git a/backend/app/modules/generator/gencode/gen_util.py b/backend/app/modules/generator/gencode/gen_util.py index 1d69b546..7e48bcf7 100644 --- a/backend/app/modules/generator/gencode/gen_util.py +++ b/backend/app/modules/generator/gencode/gen_util.py @@ -296,14 +296,17 @@ class GenUtils: @classmethod def split_column_type(cls, column_type: str) -> list[str]: - """拆分列类型 + """拆分列类型中的长度/精度参数（**只返回校验过的数字**）。 + + 原实现用 ``column_type.split("(")[1]`` 直接切分，会把括号内的任意文本带出去； + 这里改为正则捕获 + 数字校验（与 template_safety.extract_type_params 同源规则）。 参数: - column_type (str): 字段类型。 返回: - - list[str]: 拆分结果。 + - list[str]: 形如 ["64"] 或 ["10", "2"]；无有效参数时返回 []。 """ - if "(" in column_type and ")" in column_type: - return column_type.split("(")[1].split(")")[0].split(",") - return [] + from .template_safety import extract_type_params + + return list(extract_type_params(column_type)) diff --git a/backend/app/modules/generator/gencode/jinja2_template_util.py b/backend/app/modules/generator/gencode/jinja2_template_util.py index 953b111c..960c9328 100644 --- a/backend/app/modules/generator/gencode/jinja2_template_util.py +++ b/backend/app/modules/generator/gencode/jinja2_template_util.py @@ -1,3 +1,4 @@ +import copy import re from datetime import datetime from typing import Any @@ -15,6 +16,24 @@ from .schema import ( GenTableColumnOutSchema, GenTableOutSchema, ) +from .template_safety import ( + column_type_base, + escape_template_text, + html_attr, + pick_sqlalchemy_type, + safe_ident, + safe_python_type, + vue_label, +) + +# 哨兵文本：用于「用哨兵值渲染同一模板，得到结构骨架」的落盘前比对 +# （见 template_safety.assert_python_structure / assert_text_structure） +SENTINEL_TEXT = "__SAFE_TEXT__" + +# 自由文本字段：哨兵化只针对它们（保证结构比对有意义）；标识符/类型/开关/长度等 +# 必须保持真实值，否则模板分支会与骨架不一致，产生误报 +_SENTINEL_TABLE_TEXT_FIELDS = ("table_name", "table_comment", "function_name", "business_name", "sub_table_name", "description") +_SENTINEL_COLUMN_TEXT_FIELDS = ("column_comment", "dict_type") class Jinja2TemplateUtil: @@ -75,6 +94,14 @@ class Jinja2TemplateUtil: "snake_to_camel": CamelCaseUtil.snake_to_camel, "get_sqlalchemy_type": cls.get_sqlalchemy_type, "python_to_ts_type": cls.python_type_to_ts_type, + # 模板内按上下文安全化（详见 template_safety.py）： + # 绝大多数值已在 prepare_context 统一处理，模板里可按需显式调用 + "py_text": escape_template_text, + "py_ident": safe_ident, + "py_type": safe_python_type, + "vue_label": vue_label, + # 静态 HTML 属性值：实体转义（展示保真，不用 \uXXXX） + "html_attr": html_attr, }, ) return cls._env @@ -146,6 +173,194 @@ class Jinja2TemplateUtil: segs.append(seg) return "/".join(segs) if segs else "entity" + @classmethod + def safe_render_schema(cls, gen_table: GenTableOutSchema) -> GenTableOutSchema: + """构造「仅用于模板渲染」的安全副本。 + + 生成物是会被执行/被构建链消费的源码（``.py`` 会被 ``app/core/discover.py`` 启动时 import）， + 因此所有用户可控文本必须按所处上下文安全化后才能进模板。此处统一处理， + 避免逐个模板/逐个字段遗漏；``autoescape=True`` 不适用（会把生成的代码本身转义坏）。 + + **从 schema 进入模板的值按四类显式处理**（实现见 template_safety.py）： + 1. 字符串类（注释/描述/功能名/表名…）→ ``escape_template_text``（``\\uXXXX``，仅用于字符串/注释上下文）； + 2. 标识符类（列名/类名/模块名/python_field）→ ``safe_ident``（含关键字与保留字排除）； + 3. 代码位置类（SQLAlchemy 类型、长度/精度）→ ``pick_sqlalchemy_type``：**只从常量白名单派生**，不接收自由文本； + 4. 路径类（包名/模块名/业务路径/文件名）→ slug/白名单规范化（``business_name_to_path`` 等），不含 ``..`` 与分隔符。 + 展示位置（``.vue`` 文本节点与静态属性）另有 ``vue_label``/``html_attr``：HTML 实体转义，保真且不破坏标记结构。 + + 参数: + - gen_table (GenTableOutSchema): 原始生成配置（不会被修改）。 + + 返回: + - GenTableOutSchema: 深拷贝并按上下文安全化后的副本。 + """ + safe_table = copy.deepcopy(gen_table) + cls._sanitize_schema_in_place(safe_table, set()) + return safe_table + + @classmethod + def sentinel_render_schema(cls, gen_table: GenTableOutSchema) -> GenTableOutSchema: + """构造「哨兵值」副本：自由文本换成固定哨兵（空值保持为空），用于生成结构骨架。 + + 与 :meth:`safe_render_schema` 的区别：这里不追求可展示/可用，只要求**结构中性**—— + 值里不含任何可能改变模板分支或语法结构的字符。用它渲染同一模板得到的 AST/行结构 + 即为「该模板的规范骨架」，落盘前与真实产物比对（见 template_safety 的结构闸门）。 + + 参数: + - gen_table (GenTableOutSchema): 已安全化的渲染配置。 + + 返回: + - GenTableOutSchema: 自由文本字段被替换为哨兵的深拷贝。 + """ + sentinel_table = copy.deepcopy(gen_table) + cls._sentinelize_schema_in_place(sentinel_table, set()) + return sentinel_table + + @classmethod + def _sentinelize_schema_in_place(cls, table: GenTableOutSchema, seen: set[int]) -> None: + """就地哨兵化自由文本字段（标识符/类型/开关/长度保持不变），带环保护。""" + if table is None or id(table) in seen: + return + seen.add(id(table)) + for field in _SENTINEL_TABLE_TEXT_FIELDS: + if hasattr(table, field): + setattr(table, field, _sentinelize_value(getattr(table, field))) + for column in table.columns or []: + cls._sentinelize_column_in_place(column, seen) + if table.pk_column is not None: + cls._sentinelize_column_in_place(table.pk_column, seen) + if table.sub_table is not None: + cls._sentinelize_schema_in_place(table.sub_table, seen) + + @staticmethod + def _sentinelize_column_in_place(column: GenTableColumnOutSchema, seen: set[int]) -> None: + """就地哨兵化字段级自由文本（列名/类型/长度等保持原值）。""" + if column is None or id(column) in seen: + return + seen.add(id(column)) + for field in _SENTINEL_COLUMN_TEXT_FIELDS: + if hasattr(column, field): + setattr(column, field, _sentinelize_value(getattr(column, field))) + + @classmethod + def iter_free_text_values(cls, gen_table: GenTableOutSchema) -> list[str]: + """枚举本次渲染里会进入模板的**自由文本值**（与哨兵化字段一一对应，按同样顺序）。 + + 供 ``.vue``/``.ts`` 的行结构比对使用：把真实值与哨兵都抹成占位符后再逐行比较。 + + 参数: + - gen_table (GenTableOutSchema): 已安全化的渲染配置。 + + 返回: + - list[str]: 非空自由文本值列表。 + """ + values: list[str] = [] + seen: set[int] = set() + + def walk(table: GenTableOutSchema) -> None: + if table is None or id(table) in seen: + return + seen.add(id(table)) + for field in _SENTINEL_TABLE_TEXT_FIELDS: + text = getattr(table, field, None) + if text: + values.append(str(text)) + for column in table.columns or []: + collect_column(column) + if table.pk_column is not None: + collect_column(table.pk_column) + if table.sub_table is not None: + walk(table.sub_table) + + def collect_column(column: GenTableColumnOutSchema) -> None: + if column is None or id(column) in seen: + return + seen.add(id(column)) + for field in _SENTINEL_COLUMN_TEXT_FIELDS: + text = getattr(column, field, None) + if text: + values.append(str(text)) + + walk(gen_table) + return values + + @classmethod + def iter_render_dynamic_values(cls, gen_table: GenTableOutSchema) -> list[str]: + """返回真实产物里可能出现的**所有动态文本形态**（原值 / 转义值 / 展示形态）。 + + 供 ``.vue``/``.ts`` 的行结构比对把动态部分抹成占位符：同一个值可能以 + ``escape_template_text``（JS 字符串）、``html_attr``（静态属性）、``vue_label``（文本节点） + 三种形态出现，三者都要抹掉，否则会与骨架产生假阳性。 + + 参数: + - gen_table (GenTableOutSchema): 已安全化的渲染配置。 + + 返回: + - list[str]: 动态文本形态列表。 + """ + tokens: list[str] = [] + for value in cls.iter_free_text_values(gen_table): + text = str(value) + tokens.extend([text, escape_template_text(text), html_attr(text), vue_label(text)]) + return tokens + + @classmethod + def _sanitize_schema_in_place(cls, table: GenTableOutSchema, seen: set[int]) -> None: + """就地安全化一张表（表级文本 + 全部字段 + 子表），带环保护。 + + 参数: + - table (GenTableOutSchema): 待处理的安全副本（会被修改）。 + - seen (set[int]): 已处理对象 id 集合（主/子表互相引用时防重入）。 + + 返回: + - None + """ + if table is None or id(table) in seen: + return + seen.add(id(table)) + + # 落进字符串字面量/注释/文档字符串的文本 + table.table_name = escape_template_text(table.table_name) + table.table_comment = escape_template_text(table.table_comment) + table.function_name = escape_template_text(table.function_name) + # 落进标识符位置的值（类名 / 模块名 / 包名 / 外键列名） + table.class_name = safe_ident(table.class_name, default="") + table.module_name = safe_ident(table.module_name, default="") + table.package_name = safe_ident(table.package_name, default="") + table.sub_table_fk_name = safe_ident(table.sub_table_fk_name, default="") + # 业务名允许 a/b 多段，且参与目录派生，按文本处理即可（派生见 prepare_context） + table.business_name = escape_template_text(table.business_name) + table.sub_table_name = escape_template_text(table.sub_table_name) + + for column in table.columns or []: + cls._sanitize_column_in_place(column, seen) + # pk_column 通常就是 columns 里的同一个对象（见 set_pk_column），必须靠 seen 去重， + # 否则同一字段被转义两次（\u0027 → \u005cu0027），产物里的文本会变形 + if table.pk_column is not None: + cls._sanitize_column_in_place(table.pk_column, seen) + if table.sub_table is not None: + cls._sanitize_schema_in_place(table.sub_table, seen) + + @staticmethod + def _sanitize_column_in_place(column: GenTableColumnOutSchema, seen: set[int]) -> None: + """就地安全化单个字段：列名/类型注解按标识符处理，注释/字典类型按文本转义。 + + 参数: + - column (GenTableColumnOutSchema): 待处理的安全副本（会被修改）。 + - seen (set[int]): 已处理对象 id 集合（同一字段被 columns 与 pk_column 同时引用时防重复转义）。 + + 返回: + - None + """ + if column is None or id(column) in seen: + return + seen.add(id(column)) + column.column_name = safe_ident(column.column_name, default="") + column.python_type = safe_python_type(column.python_type) + column.python_field = safe_ident(column.python_field, default="") + column.column_comment = escape_template_text(column.column_comment) + column.dict_type = escape_template_text(column.dict_type) + @classmethod def prepare_context(cls, gen_table: GenTableOutSchema) -> dict[str, Any]: """准备模板变量。 @@ -159,6 +374,13 @@ class Jinja2TemplateUtil: # 处理options为None的情况 # if not gen_table.options: # raise ValueError('请先完善生成配置信息') + # 目录/文件名派生值仍按原始业务名计算，保证既有产物路径不变； + # 其余进入模板的内容一律取安全副本（详见 template_safety.py）。 + business_name_raw = (gen_table.business_name or "").strip() + business_path = cls.business_name_to_path(business_name_raw) + business_name_slug = cls.business_name_to_slug(business_name_raw) + gen_table = cls.safe_render_schema(gen_table) + class_name = gen_table.class_name or "" package_name = (gen_table.package_name or "").strip() module_name = (gen_table.module_name or "").strip() @@ -169,8 +391,6 @@ class Jinja2TemplateUtil: # - 分系统根：package_name = module_xxx # - 目录固定为：module_xxx / module_name（不再额外使用业务名作为目录层级） # - 权限前缀固定为：module_xxx:module_name（操作在模板里再拼 :query/:create...） - business_path = cls.business_name_to_path(business_name) - business_name_slug = cls.business_name_to_slug(business_name) permission_prefix = ":".join([s for s in [package_name, module_name] if s]) api_route_prefix = cls.get_api_route_prefix(package_name) @@ -262,17 +482,20 @@ class Jinja2TemplateUtil: - dict[str, Any]: 子表模板上下文字典。 """ ctx = cls.prepare_context(sub) - scn = (sub.class_name or GenUtils.convert_class_name(sub.table_name or "")).strip() + # 父表字段同样进入模板（parent_* / relationship 实参），必须取自安全副本 + parent_safe = cls.safe_render_schema(parent) + sub_safe = cls.safe_render_schema(sub) + scn = (sub_safe.class_name or GenUtils.convert_class_name(sub_safe.table_name or "")).strip() ctx["is_sub_entity"] = True - ctx["parent_class_name"] = parent.class_name or "" - ctx["parent_model_class_name"] = f"{parent.class_name}Model" - ctx["parent_table_name"] = parent.table_name or "" - ctx["parent_pk_column_name"] = (parent.pk_column.column_name if parent.pk_column else None) or "id" - ctx["parent_rel_name"] = SnakeCaseUtil.camel_to_snake(parent.class_name or "parent") + ctx["parent_class_name"] = parent_safe.class_name or "" + ctx["parent_model_class_name"] = f"{parent_safe.class_name}Model" + ctx["parent_table_name"] = parent_safe.table_name or "" + ctx["parent_pk_column_name"] = (parent_safe.pk_column.column_name if parent_safe.pk_column else None) or "id" + ctx["parent_rel_name"] = SnakeCaseUtil.camel_to_snake(parent_safe.class_name or "parent") ctx["parent_list_rel_name"] = f"{SnakeCaseUtil.camel_to_snake(scn)}_list" - ctx["sub_table_fk_name"] = (parent.sub_table_fk_name or "").strip() - ctx["model_import_list"] = cls.get_model_import_list(sub, is_sub_entity=True) - ctx["schema_import_list"] = cls.get_schema_import_list(sub) + ctx["sub_table_fk_name"] = (parent_safe.sub_table_fk_name or "").strip() + ctx["model_import_list"] = cls.get_model_import_list(sub_safe, is_sub_entity=True) + ctx["schema_import_list"] = cls.get_schema_import_list(sub_safe) return ctx @classmethod @@ -529,10 +752,9 @@ class Jinja2TemplateUtil: if "[]" in column_type: return "array" - # 提取基本类型 - if "(" in column_type: - return column_type.split("(")[0] - return column_type + # 提取基本类型：走严格基名提取（只允许 [A-Za-z][A-Za-z0-9_ ]*，剔除 unsigned 等修饰词）， + # 不直接 split 原始文本，避免把括号前的任意内容当作类型名带出去 + return column_type_base(column_type) @classmethod def merge_same_imports(cls, imports: list[str], import_start: str) -> list[str]: @@ -679,81 +901,39 @@ class Jinja2TemplateUtil: @classmethod def get_sqlalchemy_type(cls, column: Any) -> str: - """获取 SQLAlchemy 类型。 + """获取 SQLAlchemy 类型（**代码位置** → 一律由常量白名单派生，见 template_safety）。 + + 类型名只能来自 ``GenConstant.DB_TO_SQLALCHEMY`` 的值，长度/精度只能来自 + ``column_length.isdigit()`` 或 ``column_type`` 括号内的**数字捕获**； + 不再使用 ``column_type.split("(")[1]`` 这类字符串切分（会把任意内容带进产物）。 参数: - column (Any): 列对象或列类型字符串。 返回: - - str: SQLAlchemy 类型字符串。 + - str: SQLAlchemy 类型字符串（如 ``String(64)`` / ``Numeric(10,2)`` / ``DateTime``）。 """ - # 获取column_type和column_length - column_type = column - column_length = None - - # 检查是否是对象 + column_type: Any = column + column_length: Any = None if hasattr(column, "column_type"): - column_type = column.column_type or "" - column_length = column.column_length or None + column_type = getattr(column, "column_type", "") or "" + column_length = getattr(column, "column_length", None) or None + # 先按既有口径去掉 COLLATE / UNSIGNED，便于与常量表的键匹配 column_type = cls.normalize_db_column_type_for_mapping(column_type) + return pick_sqlalchemy_type(column_type, column_length) - # MySQL：仅 tinyint(1) 映射为 Boolean；其余 tinyint 走 SmallInteger（见 GenConstant.DB_TO_SQLALCHEMY） - ct_norm = (column_type or "").strip() - if settings.DATABASE_TYPE != "postgres" and ct_norm: - ct_lower = ct_norm.lower() - if ct_lower.startswith("tinyint(1)"): - return "Boolean" - # 首先尝试匹配完整类型（包括括号） - sqlalchemy_type = StringUtil.get_mapping_value_by_key_ignore_case(GenConstant.DB_TO_SQLALCHEMY, column_type) +def _sentinelize_value(value: Any) -> Any: + """把非空文本替换为哨兵文本（空值/None 原样保留，避免改变模板的真值分支）。 - # 特殊处理PostgreSQL类型 - if settings.DATABASE_TYPE == "postgres": - if column_type.upper() == "BOOLEAN": - return "Boolean" - if column_type.upper() == "REAL" or column_type.upper() == "DOUBLE PRECISION": - return "Float" - if column_type.upper() == "TIMESTAMP": - return "DateTime" - if column_type.upper() == "JSONB": - return "JSONB" - if column_type.upper() == "UUID": - return "Uuid" - if column_type.upper() == "BYTEA": - return "LargeBinary" + 参数: + - value (Any): 原始值。 - # get_mapping_value_by_key_ignore_case 未命中时返回 ""，须与 None 同样视为未匹配 - if not sqlalchemy_type and "(" in column_type: - # 如果没有匹配到，再尝试剥离括号 - column_type_list = column_type.split("(") - col_type = column_type_list[0] - # 将 'character' 映射为 'char' 以匹配常量定义 - if col_type.lower() == "character": - col_type = "char" - sqlalchemy_type = StringUtil.get_mapping_value_by_key_ignore_case(GenConstant.DB_TO_SQLALCHEMY, col_type) - # 如果是字符串类型且包含括号参数，保持原参数 - if sqlalchemy_type in ["String", "CHAR"] or sqlalchemy_type in ["Numeric", "DECIMAL"]: - sqlalchemy_type += "(" + column_type_list[1] - elif not sqlalchemy_type: - # 处理没有括号的类型 - col_type = column_type - # 将 'character' 映射为 'char' 以匹配常量定义 - if col_type.lower() == "character": - col_type = "char" - sqlalchemy_type = StringUtil.get_mapping_value_by_key_ignore_case(GenConstant.DB_TO_SQLALCHEMY, col_type) - # 如果是字符串类型且没有指定长度，使用column_length或默认255 - if sqlalchemy_type in ["String", "CHAR"]: - length = column_length if column_length and column_length.isdigit() else "255" - sqlalchemy_type += f"({length})" - # 对于已经匹配到的类型，如果是字符串类型且column有长度信息，添加长度 - elif sqlalchemy_type in ["String", "CHAR"] and "(" not in sqlalchemy_type: - # 检查column_length是否有效 - length = column_length if column_length and column_length.isdigit() else "255" - sqlalchemy_type += f"({length})" - - # 如果没有找到匹配的类型，使用String(column_length)或String(255)作为默认类型 - if not sqlalchemy_type: - length = column_length if column_length and column_length.isdigit() else "255" - sqlalchemy_type = f"String({length})" - return sqlalchemy_type + 返回: + - Any: 哨兵文本或原值。 + """ + if value is None: + return None + text = str(value) + return SENTINEL_TEXT if text else text diff --git a/backend/app/modules/generator/gencode/schema.py b/backend/app/modules/generator/gencode/schema.py index 240df6f7..8bf46110 100644 --- a/backend/app/modules/generator/gencode/schema.py +++ b/backend/app/modules/generator/gencode/schema.py @@ -4,6 +4,8 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator from app.core.base_schema import BaseQueryParam, BaseSchema, UserByQueryParam, UserBySchema +from .template_safety import is_safe_identifier, is_valid_column_type, normalize_column_type, reserved_word_reason + class GenDBTableSchema(BaseModel): """数据库中的表信息（跨方言统一结构）。 @@ -59,6 +61,71 @@ class GenTableColumnSchema(BaseModel): dict_type: str | None = Field(default="", description="字典类型") sort: int = Field(default=0, description="排序") + @field_validator("column_name", mode="before") + @classmethod + def validate_column_name(cls, v: str | None) -> str: + """列名白名单校验：必须能直接充当 Python / JS 标识符。 + + 列名会被原样写进 ``Mapped[..]`` 属性名、``class`` 字段名与前端 ``prop``， + 是注入面之一（见 backend/audit-backend.md B1）。非法名（含引号/空格/换行/数字开头等） + 在此直接拒绝，避免生成出语法不合法或可被注入的代码。 + + 参数: + - v (str | None): 原始列名。 + + 返回: + - str: 去空白后的合法列名；未填写时返回空串（沿用历史行为）。 + + 异常: + - ValueError: 列名不是合法标识符时抛出。 + """ + if v is None: + return "" + text = str(v).strip() + if not text: + return "" + if not is_safe_identifier(text): + reason = reserved_word_reason(text) + if reason: + raise ValueError( + f"列名 {text!r} 是 {reason}，作为字段名会生成非法代码（如 class: Mapped[...]），" + f"请改名后再导入/保存（可改为 {text}_ 之类）", + ) + raise ValueError(f"列名 {text!r} 不是合法的标识符（仅允许字母/数字/下划线，且不能以数字开头），代码生成器无法为它生成合法代码，请先改名") + return text + + @field_validator("column_type", mode="before") + @classmethod + def validate_column_type(cls, v: str | None) -> str: + """列类型形状白名单校验（结构化，非法即拒绝）。 + + 允许的形状：``基名`` + 可选 ``(数字)`` / ``(数字,数字)`` + 可选 ``[]`` + 可选 ``unsigned``。 + MySQL 的 ``enum(...)``/``set(...)`` 会先归一化为基名 ``enum``/``set``（其成员列表既无用途也含引号）。 + + 为什么必须校验：列类型会参与「代码位置」派生（SQLAlchemy 类型名 + 长度/精度）， + 任意文本一旦被拼接进生成代码就等于注入面（见 backend/audit-backend.md B1）。 + + 参数: + - v (str | None): 原始列类型。 + + 返回: + - str: 归一化后的列类型；未填写时返回空串（由 SQLAlchemy 映射回落 String）。 + + 异常: + - ValueError: 形状非法时抛出（报错带上列类型原文）。 + """ + if v is None: + return "" + text = str(v).strip() + if not text: + return "" + if not is_valid_column_type(text): + raise ValueError( + f"列类型 {text!r} 形状非法：仅允许「类型名 + 可选(长度[,精度]) + 可选[] + 可选 unsigned」，例如 varchar(64)、decimal(10,2)、tinyint(1) unsigned；" + "含引号/分号/括号内非数字等内容会被拒绝（代码生成器不接收自由文本类型）", + ) + return normalize_column_type(text) + class GenTableColumnOutSchema(GenTableColumnSchema, BaseSchema): """业务表字段输出模型 diff --git a/backend/app/modules/generator/gencode/service.py b/backend/app/modules/generator/gencode/service.py index b9083ea6..3af9d6bf 100644 --- a/backend/app/modules/generator/gencode/service.py +++ b/backend/app/modules/generator/gencode/service.py @@ -33,7 +33,7 @@ from app.utils.common_util import CamelCaseUtil, compute_menu_route_first_segmen from .crud import GenTableColumnCRUD, GenTableCRUD from .gen_util import GenUtils -from .jinja2_template_util import Jinja2TemplateUtil +from .jinja2_template_util import SENTINEL_TEXT, Jinja2TemplateUtil from .schema import ( GenSyncColumnChange, GenSyncPreviewSchema, @@ -43,6 +43,7 @@ from .schema import ( GenTableQueryParam, GenTableSchema, ) +from .template_safety import assert_python_structure, assert_text_structure def handle_service_exception(func: Callable) -> Callable: @@ -61,7 +62,9 @@ def handle_service_exception(func: Callable) -> Callable: except CustomException: raise except Exception as e: - raise CustomException(msg=f"{func.__name__}执行失败: {e!s}") + # 服务层公共出口：这里捕获到的都是**非业务**异常（模板渲染/DB/反射等内部故障）， + # 按 500 返回，避免监控把服务端故障误判为客户端错误；业务异常由上面的分支原样透传（保留自身 4xx） + raise CustomException(msg=f"{func.__name__}执行失败: {e!s}", status_code=500) return wrapper @@ -370,7 +373,7 @@ class GenTableService: except CustomException: raise except Exception as e: - raise CustomException(msg=f"导入失败, {e!s}") + raise CustomException(msg=f"导入失败, {e!s}", status_code=500) @handle_service_exception async def create_table(self, sql: str) -> bool | None: @@ -454,7 +457,7 @@ class GenTableService: except CustomException: raise except Exception as e: - raise CustomException(msg=f"创建表结构失败: {e!s}") + raise CustomException(msg=f"创建表结构失败: {e!s}", status_code=500) @handle_service_exception async def update_gen_table(self, data: GenTableSchema, table_id: int) -> GenTableOutSchema: @@ -516,7 +519,7 @@ class GenTableService: except CustomException: raise except Exception as e: - raise CustomException(msg=str(e)) + raise CustomException(msg=str(e), status_code=500) else: raise CustomException(msg="业务表不存在") @@ -541,7 +544,7 @@ class GenTableService: # 再删除表信息 await GenTableCRUD(self.auth, self.db).delete_gen_table(ids) except Exception as e: - raise CustomException(msg=str(e)) + raise CustomException(msg=str(e), status_code=500) @handle_service_exception async def get_gen_table_by_id(self, table_id: int) -> GenTableOutSchema: @@ -660,12 +663,40 @@ class GenTableService: raise CustomException(msg="包名不能为空") # 1. 先写代码文件（风险最高，放最前，失败不产生菜单孤儿数据） - async def _write_templates(templates: list[str], ctx: dict[str, Any], table_schema: GenTableOutSchema) -> None: + repo_root = BASE_DIR.parent.resolve() + + async def _write_templates( + templates: list[str], + ctx: dict[str, Any], + table_schema: GenTableOutSchema, + *, + reference_ctx: dict[str, Any], + dynamic_values: list[str], + ) -> None: for template in templates: try: - render_content = await env.get_template(template).render_async(**ctx) + tmpl = env.get_template(template) + render_content = await tmpl.render_async(**ctx) + # 同一模板用「哨兵值」再渲染一次，得到该模板的规范结构骨架 + reference_content = await tmpl.render_async(**reference_ctx) file_name = Jinja2TemplateUtil.get_file_name(template, table_schema) full_path = BASE_DIR.parent.joinpath(file_name) + # 落盘前三重校验（详见 gencode/template_safety.py）： + # ① 目标路径必须仍在仓库根内，防止路径穿越改写仓库外的文件； + # ② .py 产物必须语法合法，且**结构与哨兵骨架一致**（仅 ast.parse 发现不了注入）； + # ③ .vue/.ts/.toml 退化为行结构比对，禁止出现骨架之外的语句/标签结构。 + if not full_path.resolve().is_relative_to(repo_root): + raise CustomException(msg=f"生成路径越出仓库范围，已拒绝写入：{file_name}") + if file_name.endswith(".py"): + assert_python_structure(render_content, reference_content, filename=file_name) + else: + assert_text_structure( + render_content, + reference_content, + sentinel=SENTINEL_TEXT, + dynamic_values=dynamic_values, + filename=file_name, + ) gen_path = str(full_path) os.makedirs(os.path.dirname(gen_path), exist_ok=True) await anyio.Path(gen_path).write_text(render_content, encoding="utf-8") @@ -681,16 +712,30 @@ class GenTableService: os.makedirs(str(d), exist_ok=True) await anyio.Path(str(init_path)).write_text("# -*- coding: utf-8 -*-", encoding="utf-8") except Exception as e: - raise CustomException(msg=f"渲染模板失败，表名：{table_schema.table_name}，详细错误信息：{e!s}") + raise CustomException(msg=f"渲染模板失败，表名：{table_schema.table_name}，详细错误信息：{e!s}", status_code=500) - await _write_templates(render_info[0], render_info[2], gen_table_schema) + safe_schema: GenTableOutSchema = render_info[4] + reference_ctx = Jinja2TemplateUtil.prepare_context(Jinja2TemplateUtil.sentinel_render_schema(safe_schema)) + dynamic_values = Jinja2TemplateUtil.iter_render_dynamic_values(safe_schema) + await _write_templates(render_info[0], render_info[2], gen_table_schema, reference_ctx=reference_ctx, dynamic_values=dynamic_values) if gen_table_schema.sub and gen_table_schema.sub_table: gen_table_schema.sub_table.package_name = gen_table_schema.package_name if not (gen_table_schema.sub_table.module_name or "").strip(): gen_table_schema.sub_table.module_name = gen_table_schema.module_name sub_ctx = Jinja2TemplateUtil.prepare_sub_render_context(gen_table_schema, gen_table_schema.sub_table) sub_templates = Jinja2TemplateUtil.get_sub_table_template_list() - await _write_templates(sub_templates, sub_ctx, gen_table_schema.sub_table) + safe_sub = Jinja2TemplateUtil.safe_render_schema(gen_table_schema.sub_table) + sub_reference_ctx = Jinja2TemplateUtil.prepare_sub_render_context( + Jinja2TemplateUtil.sentinel_render_schema(safe_schema), + Jinja2TemplateUtil.sentinel_render_schema(safe_sub), + ) + await _write_templates( + sub_templates, + sub_ctx, + gen_table_schema.sub_table, + reference_ctx=sub_reference_ctx, + dynamic_values=dynamic_values, + ) # 2. 代码成功写入后，再创建菜单（避免失败时产生孤儿菜单数据） menu_crud = MenuCRUD(self.auth, self.db) @@ -970,7 +1015,7 @@ class GenTableService: if sub_cfg: await self.sync_db(sn, _sync_sub=False) except Exception as e: - raise CustomException(msg=f"同步失败: {e!s}") + raise CustomException(msg=f"同步失败: {e!s}", status_code=500) async def hydrate_sub_table(self, gen_table: GenTableOutSchema) -> None: """主子表：优先使用已导入的子表配置，否则回退为只读 DB 结构。 @@ -1247,6 +1292,8 @@ class GenTableService: await self.hydrate_sub_table(gen_table) self._assert_master_sub_config_valid(gen_table) context = Jinja2TemplateUtil.prepare_context(gen_table) + # 安全副本：与 prepare_context 内部使用同一函数得到，专门供「落盘前结构闸门」派生哨兵骨架与真实值集合 + safe_table = Jinja2TemplateUtil.safe_render_schema(gen_table) template_list = Jinja2TemplateUtil.get_template_list() output_files = [Jinja2TemplateUtil.get_file_name(template, gen_table) for template in template_list] - return [template_list, output_files, context, gen_table] + return [template_list, output_files, context, gen_table, safe_table] diff --git a/backend/app/modules/generator/gencode/template_safety.py b/backend/app/modules/generator/gencode/template_safety.py new file mode 100644 index 00000000..33d5c693 --- /dev/null +++ b/backend/app/modules/generator/gencode/template_safety.py @@ -0,0 +1,621 @@ +"""代码生成模板的输入安全处理。 + +背景（见 backend/audit-backend.md 的 B1） +====================================== +代码生成器把「表/列元数据」渲染成 ``.py`` / ``.vue`` / ``.ts`` / ``.toml`` 源文件，产物会被写进 +仓库，其中 ``backend/app/plugin/**`` 会在应用启动时被 ``app/core/discover.py`` 逐个 +``import_module`` 执行。因此任何用户可控文本（表注释、列注释、字段名、功能名……）都不能原样 +进入模板——否则「有代码生成权限」就等于「能在服务器上执行任意代码」。 + +为什么不用 ``autoescape=True`` +============================= +生成物的目标是**代码**而不是 HTML：全局开启 autoescape 会把 ``<`` ``>`` ``&`` 变成 ``<`` 等， +直接把生成出来的 Python / Vue 源码写坏。正确做法是**按上下文处理**： + +- 落进「字符串字面量 / 注释 / 文档字符串」的文本 → :func:`escape_template_text` + （转义 ``\\ ' " 换行 反引号`` 等，Python / JS / TOML 的字符串都会把它解析回原字符，保真； + 但源码层面不再出现能闭合字面量或换行的字符，因而无法逃出当前语法上下文）。 +- 落进「标识符位置」的值（列名、类名、类型注解……）→ :func:`safe_ident`。 +- 落进 ``.vue`` 模板「文本节点」的展示文本 → :func:`vue_label`（额外去掉尖括号与花括号）。 + +两层防线 +======== +1. 口径层：``gencode/schema.py`` 对 ``column_name`` 做白名单校验（拒绝非法标识符）； +2. 渲染层：本模块的转义在 :meth:`Jinja2TemplateUtil.prepare_context` 统一施加， + 即使数据来自物理库反射（绕过表单校验）也不会产出可执行注入。 +""" + +import ast +import keyword +import re + +from app.common.constant import GenConstant + +# 需要转义的字符 → \uXXXX 形式。 +# 该形式在 Python 字符串（含三引号文档字符串）、JS 字符串 / 模板字符串、TOML basic string +# 中都会被解析回原字符，因此既安全又保真。 +_ESCAPE_MAP: dict[str, str] = { + "\\": "\\u005c", # 反斜杠必须先处理，避免二次转义 + "'": "\\u0027", # Python 单引号 / JS 单引号 + '"': "\\u0022", # Python 双引号 / JS 双引号 / TOML basic string 定界符 + "\n": "\\u000a", + "\r": "\\u000d", + "\t": "\\u0009", + "\v": "\\u000b", + "\f": "\\u000c", + "\0": "\\u0000", + "`": "\\u0060", # JS 模板字符串定界符 + # U+2028/U+2029（行/段分隔符）：在 JS 里被当作换行，可用来断行逃逸；在 Python 字符串里合法 + "\u2028": "\\u2028", + "\u2029": "\\u2029", +} + +# 其余控制字符：C0（换行/制表等已在 _ESCAPE_MAP 中显式处理）+ DEL(7F) + C1 控制符(80-9F) +_CONTROL_RE = re.compile(r"[\x01-\x08\x0e-\x1f\x7f-\x9f]") + +# \uXXXX 转义的反解（展示位置使用） +_UNESCAPE_RE = re.compile(r"\\u([0-9a-fA-F]{4})") + +# JS 模板字符串插值起始序列：$ 后紧跟 { 时必须打断，否则 ${...} 会被当成表达式执行 +_JS_INTERPOLATION_RE = re.compile(r"\$\{") + +# 合法标识符（Python / JS 通用子集） +_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + +# 标识符规范化：去掉所有非 [A-Za-z0-9_] +_IDENT_STRIP_RE = re.compile(r"[^A-Za-z0-9_]") + +# Vue 文本节点额外需要剔除的字符：尖括号与花括号会改变标记结构 / 触发 Vue 插值 +_VUE_LABEL_STRIP_RE = re.compile(r"[<>{}]") + +# HTML 文本/属性位置的实体转义（保真：浏览器渲染回原字符，而不是把 \uXXXX 当字面量显示） +_HTML_ESCAPE_MAP: dict[str, str] = { + "&": "&", + "<": "<", + ">": ">", + '"': """, + "'": "'", +} + +# 标识符长度上限（MySQL 标识符上限 64） +_IDENTIFIER_MAX_LENGTH = 64 + +# Python 关键字/软关键字：作为字段名会直接语法错误（如 column_name="class"） +_PYTHON_RESERVED: frozenset[str] = frozenset(keyword.kwlist) | frozenset(keyword.softkwlist) + +# JS/TS 保留字与字面量：作为裸标识符会语法错误或改变语义 +_JS_RESERVED: frozenset[str] = frozenset( + { + "abstract", "arguments", "await", "boolean", "break", "byte", "case", "catch", "char", "class", + "const", "continue", "debugger", "default", "delete", "do", "double", "else", "enum", "eval", + "export", "extends", "false", "final", "finally", "float", "for", "function", "goto", "if", + "implements", "import", "in", "instanceof", "int", "interface", "let", "long", "native", "new", + "null", "package", "private", "protected", "public", "return", "short", "static", "super", + "switch", "synchronized", "this", "throw", "throws", "transient", "true", "try", "typeof", + "undefined", "var", "void", "volatile", "while", "with", "yield", + }, +) + +# 保留字命中时的前缀（保证仍是合法标识符，同时避免与关键字冲突） +_RESERVED_PREFIX = "_" + + +def escape_template_text(value: object) -> str: + """把任意文本转义为「放进字符串字面量/注释/文档字符串都安全」的形式。 + + 参数: + - value (object): 原始值（None 视为空串）。 + + 返回: + - str: 转义后的文本；内容不含可闭合字符串字面量、可换行或可开启 JS 插值的字符。 + """ + if value is None: + return "" + text = str(value) + for raw, escaped in _ESCAPE_MAP.items(): + text = text.replace(raw, escaped) + text = _CONTROL_RE.sub(lambda m: f"\\u{ord(m.group()):04x}", text) + # ${ 打断为 \u0024{，避免在 JS 模板字符串里变成插值表达式 + # 注意：替换值用 lambda 返回，re.sub 的字符串替换会把 \u 当成非法转义 + return _JS_INTERPOLATION_RE.sub(lambda m: "\\u0024{", text) + + +def safe_ident(value: object, default: str = "entity") -> str: + """把任意值规范为合法标识符（用于列名、类名、类型注解等位置）。 + + 参数: + - value (object): 原始值。 + - default (str): 空值/规范化后为空的兜底标识符。 + + 返回: + - str: 形如 ``[A-Za-z_][A-Za-z0-9_]*`` 的标识符，长度不超过 64。 + """ + text = "" if value is None else str(value).strip() + if not text: + return default + # 只替换非法字符，不改动已有合法标识符（避免 a__b / _x 这类名字被“规范化”后与库中列名不一致） + text = _IDENT_STRIP_RE.sub("_", text) + if text[0].isdigit(): + text = f"_{text}" + # 关键字/保留字（Python + JS）不能作为裸标识符：加前缀而不是报错（用于类名/模块名/字段名等位置） + if text in _PYTHON_RESERVED or text in _JS_RESERVED: + text = f"{_RESERVED_PREFIX}{text}" + return text[:_IDENTIFIER_MAX_LENGTH] + + +def is_safe_identifier(value: object) -> bool: + """判断值是否本身即为合法标识符（无需规范化）。 + + 同时排除 Python 关键字与 JS/TS 保留字：它们作为裸标识符会出现语法错误 + （如列名 ``class`` 会生成 ``class: Mapped[...]``）。 + + 参数: + - value (object): 待判断的值。 + + 返回: + - bool: 合法返回 True。 + """ + if value is None: + return False + text = str(value).strip() + if not text or len(text) > _IDENTIFIER_MAX_LENGTH: + return False + if text in _PYTHON_RESERVED or text in _JS_RESERVED: + return False + return bool(_IDENTIFIER_RE.match(text)) + + +def reserved_word_reason(value: object) -> str | None: + """若值命中 Python 关键字或 JS 保留字，返回原因文案（供报错信息使用），否则 None。 + + 参数: + - value (object): 待判断的值。 + + 返回: + - str | None: 命中原因（"Python 关键字" / "JS 保留字"）。 + """ + text = "" if value is None else str(value).strip() + if text in _PYTHON_RESERVED: + return "Python 关键字" + if text in _JS_RESERVED: + return "JS/TS 保留字" + return None + + +def unescape_template_text(value: object) -> str: + """还原 :func:`escape_template_text` 产生的 ``\\uXXXX`` 转义（展示位置专用）。 + + HTML 文本/属性里不会解析 ``\\uXXXX``，若不还原就会被用户看成字面量。转义本身是**可逆**的 + （原文本里的反斜杠也被转义为 ``\\u005c``），因此这里单趟还原即可精确拿回原文。 + + 参数: + - value (object): 已转义的文本。 + + 返回: + - str: 还原后的文本。 + """ + if value is None: + return "" + return _UNESCAPE_RE.sub(lambda m: chr(int(m.group(1), 16)), str(value)) + + +def html_escape(value: object) -> str: + """HTML 实体转义（用于 ``.vue`` 的文本节点与静态属性值）。 + + 与 :func:`escape_template_text` 的区别：这里追求**展示保真**——先还原 ``\\uXXXX``（HTML 不解析该转义）， + 再用实体转义，浏览器渲染回原字符。 + + 参数: + - value (object): 文本（可能已带 ``\\uXXXX`` 转义）。 + + 返回: + - str: 实体转义后的文本。 + """ + if value is None: + return "" + text = unescape_template_text(value) + for raw, escaped in _HTML_ESCAPE_MAP.items(): + text = text.replace(raw, escaped) + return text + + +def html_attr(value: object) -> str: + """静态 HTML 属性值的安全化（还原转义 + 实体转义，保留引号语义与展示保真）。 + + 参数: + - value (object): 文本。 + + 返回: + - str: 可直接放进 ``attr="..."`` 的文本。 + """ + return html_escape(value) + + +def vue_label(value: object) -> str: + """把展示文本处理为可安全放进 ``.vue`` **文本节点**的形式。 + + 策略（与 JS/Python 字符串上下文不同，见模块文档的「按上下文四类」）： + - 先还原 ``\\uXXXX``（HTML 不解析该转义，否则用户会看到字面量）； + - 剔除 ``< > { }``：改变标记结构 / 触发 Vue 插值的字符直接去掉； + - 其余字符用 **HTML 实体**转义：浏览器渲染回原字符，展示保真。 + + 参数: + - value (object): 原始展示文本。 + + 返回: + - str: 可安全放进文本节点的文本。 + """ + if value is None: + return "" + return html_escape(_VUE_LABEL_STRIP_RE.sub("", unescape_template_text(value))) + + +def safe_python_type(value: object, default: str = "str") -> str: + """把 ``python_type`` 规范为合法类型注解片段。 + + ``python_type`` 由表单提交且直接落在 ``Mapped[{{ column.python_type }}]`` 这类注解位置， + 必须限定为标识符形态（如 ``str``/``int``/``datetime``/``Decimal``）。 + + 参数: + - value (object): 原始值。 + - default (str): 兜底类型。 + + 返回: + - str: 合法标识符形式的类型名。 + """ + return safe_ident(value, default=default) + + +def assert_python_syntax(content: str, *, filename: str = "") -> None: + """断言渲染产物是语法合法的 Python。 + + 作为「转义/白名单是否仍然完备」的最后一道网：任何模板改动导致注入或语法破损时， + 在**写盘前**就报错，而不是把坏文件写进会被启动期 import 的目录。 + + 参数: + - content (str): 渲染后的源码文本。 + - filename (str): 产物相对路径（用于报错定位）。 + + 返回: + - None + + 异常: + - SyntaxError: 产物无法被 ast.parse 解析时抛出。 + """ + try: + ast.parse(content) + except SyntaxError as exc: + raise SyntaxError(f"{filename} 渲染结果不是合法的 Python 代码：{exc.msg}（第 {exc.lineno} 行）") from exc + + +# ══════════════════════════════════════════════════════════════════════════════ +# 「代码位置」类值：一律由**常量白名单**派生，不接收自由文本 +# ══════════════════════════════════════════════════════════════════════════════ + +# column_type 的形状白名单（结构化，不含引号/分号/换行等可注入字符） +_COLUMN_TYPE_RE = re.compile( + r"^[A-Za-z][A-Za-z0-9_ ]*(\(\s*\d+(\s*,\s*\d+)?\s*\))?(\[\])*(\s+unsigned)?$", +) +# MySQL ENUM/SET 列表（合法的库类型，但含引号）：整体归一化为基名后再做形状校验 +_ENUM_SET_RE = re.compile(r"^(enum|set)\s*\(.*\)$", re.IGNORECASE) +# 从 column_type 的括号里取参数（只接受数字，绝不拼接原始子串） +_TYPE_PARAMS_RE = re.compile(r"\(\s*(\d+)(?:\s*,\s*(\d+))?\s*\)") +# 取出类型基名（字母/数字/下划线/空格，遇括号或方括号即止） +_TYPE_BASE_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_ ]*") +# 基名里需要剔除的修饰词（它们不属于类型名，参与查表会导致 miss） +_BASE_MODIFIER_RE = re.compile(r"\s+(unsigned|zerofill)$", re.IGNORECASE) + +# 映射常量值的白名单（类型名只能取自这里，杜绝自由文本进产物） +_SQLALCHEMY_TYPE_WHITELIST: frozenset[str] = frozenset(GenConstant.DB_TO_SQLALCHEMY.values()) + +# Boolean 由 MySQL tinyint(1) 规则显式产出（常量表中未必有该值），显式纳入白名单 +_SQLALCHEMY_TYPE_WHITELIST = _SQLALCHEMY_TYPE_WHITELIST | {"Boolean"} + + +def _lookup_sqlalchemy_type(base: str) -> str: + """按基名（大小写不敏感）在常量白名单中查表。 + + 参数: + - base (str): 类型基名（已小写）。 + + 返回: + - str: 命中的常量值；未命中返回空串。 + """ + for key, value in GenConstant.DB_TO_SQLALCHEMY.items(): + if str(key).lower() == base: + return str(value) + return "" + + +def normalize_column_type(column_type: object) -> str: + """归一化列类型：去空白、把 ENUM/SET 列表压成基名。 + + 参数: + - column_type (object): 原始列类型（如 ``VARCHAR(64)`` / ``ENUM('a','b')``）。 + + 返回: + - str: 归一化后的类型文本（ENUM/SET 仅保留基名）。 + """ + text = "" if column_type is None else str(column_type).strip() + if _ENUM_SET_RE.match(text): + return _TYPE_BASE_RE.match(text).group(0).strip() # type: ignore[union-attr] + return text + + +def is_valid_column_type(column_type: object) -> bool: + """列类型形状校验：只允许「基名 + 可选(n)/(n,m) + 可选[] + 可选 unsigned」。 + + 参数: + - column_type (object): 原始列类型。 + + 返回: + - bool: 合法返回 True。 + """ + text = normalize_column_type(column_type) + if not text: + return False + return bool(_COLUMN_TYPE_RE.match(text)) + + +def column_type_base(column_type: object) -> str: + """取列类型基名（用于白名单查表，不参与最终产物拼接）。 + + 参数: + - column_type (object): 原始列类型。 + + 返回: + - str: 基名（如 ``VARCHAR``）；无法识别时为空串。 + """ + text = normalize_column_type(column_type) + match = _TYPE_BASE_RE.match(text) + if not match: + return "" + base = _BASE_MODIFIER_RE.sub("", match.group(0).strip()) + return base.strip() + + +def extract_type_params(column_type: object, column_length: object = None) -> tuple[str, ...]: + """提取「长度/精度」参数，只接受纯数字。 + + 取数优先级：``column_length.isdigit()`` → ``column_type`` 括号内正则捕获的数字。 + **不对 column_type 做字符串切分拼接**（``split("(")[1]`` 会把任意内容带进产物）。 + + 参数: + - column_type (object): 列类型。 + - column_length (object): 列长度（表单/反射字段）。 + + 返回: + - tuple[str, ...]: 形如 ("64",) 或 ("10", "2")；无有效参数时为空元组。 + """ + length = "" if column_length is None else str(column_length).strip() + if length.isdigit(): + return (length,) + text = normalize_column_type(column_type) + match = _TYPE_PARAMS_RE.search(text) + if not match: + return () + return tuple(part for part in match.groups() if part is not None) + + +def pick_sqlalchemy_type(column_type: object, column_length: object = None, *, db_type: str | None = None) -> str: + """由常量白名单派生 SQLAlchemy 类型字符串（代码位置，不接收自由文本）。 + + 规则： + - 类型名只能来自 ``GenConstant.DB_TO_SQLALCHEMY`` 的**值**（未命中回落 ``String``）； + - 长度/精度只能来自 ``column_length.isdigit()`` 或 ``column_type`` 括号内的数字捕获； + - MySQL 下 ``tinyint(1)`` 保持 ``Boolean`` 语义。 + + 参数: + - column_type (object): 列类型。 + - column_length (object): 列长度。 + - db_type (str | None): 数据库类型（默认取 settings.DATABASE_TYPE）。 + + 返回: + - str: 形如 ``String(64)`` / ``Numeric(10,2)`` / ``DateTime`` 的类型字符串。 + """ + from app.config.setting import settings # 延迟导入避免模块级循环 + + dialect = (db_type or settings.DATABASE_TYPE or "").lower() + base = column_type_base(column_type) + params = extract_type_params(column_type, column_length) + lowered = base.lower() + + # MySQL：仅 tinyint(1) 映射为 Boolean（与既有行为一致） + if dialect != "postgres" and lowered == "tinyint" and params[:1] == ("1",): + return "Boolean" + + # 查表（大小写不敏感）：命中值必须落在白名单内 + mapped = _lookup_sqlalchemy_type(lowered) + if not mapped and lowered == "character": + mapped = _lookup_sqlalchemy_type("char") + if mapped not in _SQLALCHEMY_TYPE_WHITELIST: + mapped = "String" # 白名单兜底 + + # 长度/精度：仅对需要参数的字符串与数值类型附加（数字已校验） + if params: + if mapped in {"String", "CHAR"}: + return f"{mapped}({params[0]})" + if mapped in {"Numeric", "DECIMAL"} and len(params) >= 2: + return f"{mapped}({params[0]},{params[1]})" + if mapped in {"String", "CHAR"}: + return f"{mapped}(255)" + return mapped + + +# ══════════════════════════════════════════════════════════════════════════════ +# 落盘前的「结构闸门」：仅 ast.parse 不足以发现注入，必须比对骨架结构 +# ══════════════════════════════════════════════════════════════════════════════ + +# 允许出现在生成产物里的顶层语句类型（白名单）；出现其它类型即拒绝 +_ALLOWED_TOP_LEVEL_NODES: tuple[type[ast.AST], ...] = ( + ast.Import, + ast.ImportFrom, + ast.ClassDef, + ast.FunctionDef, + ast.AsyncFunctionDef, + ast.Assign, + ast.AnnAssign, + ast.Expr, + ast.If, + ast.Try, + ast.Pass, +) + + +def _node_structure(node: ast.AST) -> object: + """把 AST 节点归一化为「结构签名」：保留节点类型与层级，抹掉具体取值/标识符。 + + 参数: + - node (ast.AST): 待归一化的节点。 + + 返回: + - object: 可比较的结构签名（元组树）。 + """ + if isinstance(node, ast.Constant): + return ("Constant", type(node.value).__name__) + if isinstance(node, ast.Name): + return ("Name",) # 标识符具体名字不参与比较 + if isinstance(node, ast.Attribute): + return ("Attribute", _node_structure(node.value)) + if isinstance(node, ast.arg): + return ("arg", node.arg) # 形参名由模板决定，参与比较 + if isinstance(node, ast.keyword): + return ("keyword", node.arg, _node_structure(node.value)) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + return ( + type(node).__name__, + node.name, + tuple(_node_structure(item) for item in node.body), + tuple(_node_structure(item) for item in node.decorator_list), + tuple(_node_structure(item) for item in node.args.args), + tuple(_node_structure(item) for item in node.args.kwonlyargs), + node.args.vararg.arg if node.args.vararg else None, + node.args.kwarg.arg if node.args.kwarg else None, + ) + if isinstance(node, ast.ClassDef): + return ( + "ClassDef", + node.name, + tuple(_node_structure(item) for item in node.bases), + tuple(_node_structure(item) for item in node.body), + tuple(_node_structure(item) for item in node.decorator_list), + ) + + fields: list[object] = [type(node).__name__] + for _field_name, value in ast.iter_fields(node): + if isinstance(value, list): + fields.append(tuple(_node_structure(item) for item in value if isinstance(item, ast.AST))) + elif isinstance(value, ast.AST): + fields.append(_node_structure(value)) + else: + fields.append(type(value).__name__) + return tuple(fields) + + +def python_structure(content: str) -> tuple[object, ...]: + """解析源码并返回顶层结构签名（供骨架比对）。 + + 参数: + - content (str): Python 源码。 + + 返回: + - tuple[object, ...]: 顶层语句的结构签名序列。 + """ + tree = ast.parse(content) + return tuple(_node_structure(node) for node in tree.body) + + +def assert_python_structure(content: str, reference: str, *, filename: str = "") -> None: + """落盘前的语义/AST 闸门：真实产物必须与「哨兵值骨架」结构一致。 + + 只做 ``ast.parse`` 不能作为安全网（注入出来的代码本身语法合法）。这里要求： + 1. 产物语法合法； + 2. 顶层语句类型必须在白名单内（Import/ClassDef/AnnAssign…）； + 3. 与用哨兵值渲染同一模板得到的**骨架**逐节点结构一致（语句数量、类型、装饰器、 + 形参、关键字参数名全部一致），任何因值而新增/改写的语句都会被发现。 + + 参数: + - content (str): 真实渲染产物。 + - reference (str): 哨兵值渲染产出的骨架。 + - filename (str): 产物相对路径（报错定位）。 + + 返回: + - None + + 异常: + - SyntaxError: 语法非法或结构不一致时抛出。 + """ + try: + ast.parse(content) + except SyntaxError as exc: + raise SyntaxError(f"{filename} 渲染结果不是合法的 Python 代码：{exc.msg}（第 {exc.lineno} 行）") from exc + + real_tree = ast.parse(content) + banned = [type(node).__name__ for node in real_tree.body if not isinstance(node, _ALLOWED_TOP_LEVEL_NODES)] + if banned: + raise SyntaxError(f"{filename} 出现白名单之外的顶层语句：{sorted(set(banned))}") + + real = python_structure(content) + skeleton = python_structure(reference) + if real != skeleton: + raise SyntaxError( + f"{filename} 渲染产物结构与模板骨架不一致（疑似模板注入或转义缺口）：" + f"真实 {len(real)} 条顶层语句 / 骨架 {len(skeleton)} 条", + ) + + +_STR_LITERAL_RE = re.compile(r"\"(?:[^\"\\]|\\.)*\"|'(?:[^'\\]|\\.)*'") + + +def _normalize_text_line(line: str, tokens: tuple[str, ...]) -> str: + """把一行里的「值」抹成占位符，只留下骨架（供 .vue/.ts 的粗粒度护栏比对）。 + + 参数: + - line (str): 待归一化的行。 + - tokens (tuple[str, ...]): 需要抹掉的动态值（哨兵 + 真实值，按长度降序）。 + + 返回: + - str: 归一化后的行骨架。 + """ + normalized = line + for token in tokens: + if token: + normalized = normalized.replace(token, "«»") + return _STR_LITERAL_RE.sub(lambda m: f"{m.group(0)[0]}«»{m.group(0)[0]}", normalized) + + +def assert_text_structure( + content: str, + reference: str, + *, + sentinel: str, + dynamic_values: object = (), + filename: str = "", +) -> None: + """``.vue`` / ``.ts`` / ``.toml`` 产物的粗粒度结构护栏。 + + 这些产物没有 AST 可比，退化为「行结构一致」：把每行的字符串字面量与哨兵值抹成占位符后， + 真实产物与骨架的行数与逐行骨架必须完全一致。任何因值而新增的行（注入的 script/标签/语句） + 或在一行内跑出字符串字面量的注入都会导致不一致。 + + 参数: + - content (str): 真实渲染产物。 + - reference (str): 哨兵值渲染产出的骨架。 + - sentinel (str): 哨兵文本。 + - dynamic_values (object): 真实值集合（会出现在真实产物里的动态文本，一并抹成占位符）。 + - filename (str): 产物相对路径（报错定位）。 + + 返回: + - None + + 异常: + - ValueError: 行结构不一致时抛出。 + """ + tokens = tuple(sorted({sentinel, *[str(item) for item in dynamic_values if item]}, key=len, reverse=True)) + real_lines = content.splitlines() + ref_lines = reference.splitlines() + if len(real_lines) != len(ref_lines): + raise ValueError(f"{filename} 产物行数与模板骨架不一致（{len(real_lines)} vs {len(ref_lines)}），疑似注入") + for index, (real_line, ref_line) in enumerate(zip(real_lines, ref_lines, strict=True), 1): + if _normalize_text_line(real_line, tokens) != _normalize_text_line(ref_line, tokens): + raise ValueError(f"{filename} 第 {index} 行结构偏离模板骨架，疑似注入：{real_line.strip()[:120]!r}") diff --git a/backend/app/modules/monitor/health/schema.py b/backend/app/modules/monitor/health/schema.py index 9522ec4c..680bf660 100644 --- a/backend/app/modules/monitor/health/schema.py +++ b/backend/app/modules/monitor/health/schema.py @@ -9,3 +9,11 @@ class ServiceInfoOut(BaseModel): environment: str = Field(..., description="运行环境") db_status: int = Field(..., description="数据库状态(0:异常 1:正常)") redis_status: int = Field(..., description="Redis状态(0:异常 1:正常)") + code_exec_enabled: bool = Field( + default=False, + description="定时任务是否允许执行用户提交的代码块(SCHEDULER_ALLOW_CODE_EXEC)；关闭时节点型的代码块任务不可调度", + ) + cors_origins_configured: bool = Field( + default=True, + description="生产环境是否已显式配置 PROD_CORS_ORIGINS；false 表示生产已按安全默认拒绝全部跨域请求", + ) diff --git a/backend/app/modules/monitor/health/service.py b/backend/app/modules/monitor/health/service.py index 32dcfbae..c0015f67 100644 --- a/backend/app/modules/monitor/health/service.py +++ b/backend/app/modules/monitor/health/service.py @@ -51,4 +51,8 @@ class HealthService: environment=settings.ENVIRONMENT.value, db_status=db_status, redis_status=redis_status, + # 升级可见性：这两个开关会静默改变行为（代码块任务不可调度 / 生产拒绝全部跨域）， + # 在健康检查里显式暴露，避免运维只看到"任务不动了"而不知原因。 + code_exec_enabled=settings.SCHEDULER_ALLOW_CODE_EXEC, + cors_origins_configured=bool(settings.PROD_CORS_ORIGINS) if settings.ENVIRONMENT.value == "prod" else True, ) diff --git a/backend/app/modules/system/auth/oauth_service.py b/backend/app/modules/system/auth/oauth_service.py index d8847b3d..0607e4d2 100644 --- a/backend/app/modules/system/auth/oauth_service.py +++ b/backend/app/modules/system/auth/oauth_service.py @@ -72,7 +72,8 @@ def _require_credentials(provider: OAuthProvider) -> tuple[str, str]: else: raise CustomException(msg="不支持的 OAuth 渠道") if not cid or not sec: - raise CustomException(msg=f"{provider} OAuth 未配置（客户端密钥为空）") + # 渠道密钥为空属**部署配置缺失**：客户端无论怎么改请求都无法成功，按 500 返回以免监控误判为客户端错误 + raise CustomException(msg=f"{provider} OAuth 未配置（客户端密钥为空）", status_code=500) return cid, sec @@ -136,7 +137,7 @@ async def _http_json(method: str, url: str, **kwargs: Any) -> Any: except json.JSONDecodeError: text = r.text logger.error(f"OAuth 非 JSON 响应: {text[:500]}") - raise CustomException(msg="OAuth 接口返回异常") + raise CustomException(msg="OAuth 接口返回异常", status_code=500) async def _http_text(method: str, url: str, **kwargs: Any) -> str: diff --git a/backend/app/modules/system/auth/service.py b/backend/app/modules/system/auth/service.py index fe9f6a58..0f034a69 100644 --- a/backend/app/modules/system/auth/service.py +++ b/backend/app/modules/system/auth/service.py @@ -81,7 +81,9 @@ async def auto_register_login_user( existing = await UserCRUD(auth, db).get(username=username) if existing: return existing - raise CustomException(msg=fail_msg) + # 原始异常被吞掉（可能是 DB/唯一约束/驱动等未知服务端故障）：按 500 返回， + # 避免监控把内部故障误判为客户端错误；确实的客户端问题由 UserService 的 CustomException 透传 + raise CustomException(msg=fail_msg, status_code=500) user = await UserCRUD(auth, db).get(username=username) if not user: raise CustomException(msg=fail_msg) diff --git a/backend/app/modules/system/auth/wx_mini_service.py b/backend/app/modules/system/auth/wx_mini_service.py index 7fde473e..82a38195 100644 --- a/backend/app/modules/system/auth/wx_mini_service.py +++ b/backend/app/modules/system/auth/wx_mini_service.py @@ -67,7 +67,7 @@ async def _http_json(method: str, url: str, **kwargs: Any) -> dict: return r.json() except json.JSONDecodeError: logger.error(f"微信 API 非 JSON 响应: {r.text[:500]}") - raise CustomException(msg="微信接口返回异常") + raise CustomException(msg="微信接口返回异常", status_code=500) async def _get_access_token(redis: Redis) -> str: diff --git a/backend/app/modules/system/dict/service.py b/backend/app/modules/system/dict/service.py index 17ed9f5d..e8d1884a 100644 --- a/backend/app/modules/system/dict/service.py +++ b/backend/app/modules/system/dict/service.py @@ -121,7 +121,7 @@ class DictTypeService: logger.info(f"创建字典类型成功: {new_obj_dict}") except Exception as e: logger.error(f"创建字典类型失败: {e}") - raise CustomException(msg="同步字典类型缓存失败") from e + raise CustomException(msg="同步字典类型缓存失败", status_code=500) from e return new_obj_dict @@ -176,7 +176,7 @@ class DictTypeService: logger.info(f"更新字典类型成功并刷新缓存: {new_obj_dict}") except Exception as e: logger.error(f"更新字典类型缓存失败: {e}") - raise CustomException(msg="同步字典类型缓存失败") from e + raise CustomException(msg="同步字典类型缓存失败", status_code=500) from e return new_obj_dict @@ -215,7 +215,7 @@ class DictTypeService: logger.info(f"删除字典类型缓存: {dt}") except Exception as e: logger.error(f"删除字典类型缓存失败: {e}") - raise CustomException(msg="同步删除字典缓存失败") from e + raise CustomException(msg="同步删除字典缓存失败", status_code=500) from e await DictTypeCRUD(self.auth, self.db).delete(ids=ids) async def set_available(self, data: BatchSetAvailable) -> None: @@ -330,7 +330,7 @@ class DictDataService: except Exception as e: logger.error(f"❌️ 字典初始化过程发生错误: {e}") - raise CustomException(msg="字典数据初始化失败") from e + raise CustomException(msg="字典数据初始化失败", status_code=500) from e @staticmethod async def get_init_cache(redis: Redis, dict_type: str) -> list[dict]: @@ -376,7 +376,7 @@ class DictDataService: raise except Exception as e: logger.error(f"获取字典缓存失败: {e!s}") - raise CustomException(msg="获取字典数据失败") from e + raise CustomException(msg="获取字典数据失败", status_code=500) from e async def _refresh_dict_cache(self, redis: Redis, dict_type: str) -> None: """刷新指定字典类型的 Redis 缓存 @@ -420,7 +420,7 @@ class DictDataService: logger.info(f"创建字典数据写入缓存成功: {obj}") except Exception as e: logger.error(f"创建字典数据写入缓存失败: {e}") - raise CustomException(msg="同步字典数据缓存失败") from e + raise CustomException(msg="同步字典数据缓存失败", status_code=500) from e return DictDataOutSchema.model_validate(obj) @@ -462,7 +462,7 @@ class DictDataService: await self._refresh_dict_cache(redis, dict_type.dict_type) except Exception as e: logger.error(f"刷新旧字典缓存失败: {e}") - raise CustomException(msg="同步旧字典数据缓存失败") from e + raise CustomException(msg="同步旧字典数据缓存失败", status_code=500) from e obj = await DictDataCRUD(self.auth, self.db).update(id=id, data=data) @@ -472,7 +472,7 @@ class DictDataService: logger.info(f"更新字典数据写入缓存成功: {obj}") except Exception as e: logger.error(f"更新字典数据写入缓存失败: {e}") - raise CustomException(msg="同步字典数据缓存失败") from e + raise CustomException(msg="同步字典数据缓存失败", status_code=500) from e return DictDataOutSchema.model_validate(obj) @@ -504,7 +504,7 @@ class DictDataService: logger.info(f"删除字典数据并刷新缓存: dict_type={dt}") except Exception as e: logger.error(f"删除字典数据刷新缓存失败: {e}") - raise CustomException(msg="同步删除字典数据缓存失败") from e + raise CustomException(msg="同步删除字典数据缓存失败", status_code=500) from e await DictDataCRUD(self.auth, self.db).delete(ids=ids) async def set_available(self, data: BatchSetAvailable) -> None: diff --git a/backend/app/modules/system/params/service.py b/backend/app/modules/system/params/service.py index 0814b000..cfbf186d 100644 --- a/backend/app/modules/system/params/service.py +++ b/backend/app/modules/system/params/service.py @@ -57,10 +57,10 @@ class ParamsService: ) if not result: logger.error(f"同步配置到缓存失败: {out}") - raise CustomException(msg="同步配置到缓存失败") + raise CustomException(msg="同步配置到缓存失败", status_code=500) except Exception as e: logger.error(f"更新系统配置失败: {e}") - raise CustomException(msg="同步配置到缓存失败") from e + raise CustomException(msg="同步配置到缓存失败", status_code=500) from e return out @@ -95,7 +95,7 @@ class ParamsService: await ParamsService._sync_configs_to_redis(redis, config_obj) except Exception as e: logger.error(f"❌️ 初始化系统参数到 Redis 失败: {e}") - raise CustomException(msg="初始化系统参数到 Redis 失败") from e + raise CustomException(msg="初始化系统参数到 Redis 失败", status_code=500) from e @staticmethod async def get_init_cache(redis: Redis) -> list[dict]: diff --git a/backend/app/modules/system/user/service.py b/backend/app/modules/system/user/service.py index 54be7f97..83f8e75e 100644 --- a/backend/app/modules/system/user/service.py +++ b/backend/app/modules/system/user/service.py @@ -379,9 +379,12 @@ class UserService: result += "\n错误信息:\n" + "\n".join(error_msgs) return result + except CustomException: + # 业务异常原样透传（导入文件为空 / 缺少必要列 / 单行数据不合法等属客户端可修正，保留 400） + raise except Exception as e: logger.error(f"批量导入用户失败: {e!s}") - raise CustomException(msg=f"导入失败: {e!s}") from e + raise CustomException(msg=f"导入失败: {e!s}", status_code=500) from e async def _process_import_row( self, diff --git a/backend/app/modules/task/cronjob/job/schema.py b/backend/app/modules/task/cronjob/job/schema.py index b4e5052e..c71ddc16 100644 --- a/backend/app/modules/task/cronjob/job/schema.py +++ b/backend/app/modules/task/cronjob/job/schema.py @@ -14,6 +14,8 @@ class SchedulerStatusSchema(BaseModel): status: str = Field(..., description="调度器状态(运行中/已暂停/已停止/未知)") is_running: bool = Field(..., description="是否运行中") job_count: int = Field(..., description="调度器任务数量") + code_exec_enabled: bool = Field(default=False, description="是否允许执行用户提交的代码块(SCHEDULER_ALLOW_CODE_EXEC)") + unschedulable_count: int = Field(default=0, description="已注册但当前不可执行的任务数（>0 时列表页会给出原因）") class SchedulerJobSchema(BaseModel): @@ -24,6 +26,8 @@ class SchedulerJobSchema(BaseModel): trigger: str = Field(..., description="触发器") next_run_time: str | None = Field(default=None, description="下次运行时间") status: int = Field(..., description="任务状态(0:运行中 1:暂停中 2:已停止 3:未知)") + schedulable: bool = Field(default=True, description="当前是否可执行；false 表示已注册但执行时会被拒绝") + unschedulable_reason: str | None = Field(default=None, description="不可执行原因（含修复建议）") class SchedulerJobModifySchema(BaseModel): diff --git a/backend/app/modules/task/cronjob/job/service.py b/backend/app/modules/task/cronjob/job/service.py index f17df212..8fbcbff9 100644 --- a/backend/app/modules/task/cronjob/job/service.py +++ b/backend/app/modules/task/cronjob/job/service.py @@ -1,5 +1,6 @@ from sqlalchemy.ext.asyncio import AsyncSession +from app.config.setting import settings from app.core.ap_scheduler import SchedulerUtil from app.core.base_schema import AuthSchema, PageResultSchema from app.core.exceptions import CustomException @@ -71,25 +72,36 @@ class JobService: state = SchedulerUtil.get_scheduler_state() is_running = SchedulerUtil.is_running() jobs = SchedulerUtil.get_jobs() + # 可观测性（t15）：把"已注册但当前不可执行"的任务显式打出并计数， + # 运维不必逐个点开才知道开关调整/处理器改名后哪些任务静默失效 + unschedulable = SchedulerUtil.log_unschedulable_jobs(context="调度器状态查询") return SchedulerStatusSchema( status=JobService._SCHEDULER_STATE_MAP.get(state, "未知"), is_running=is_running, job_count=len(jobs), + code_exec_enabled=settings.SCHEDULER_ALLOW_CODE_EXEC, + unschedulable_count=len(unschedulable), ) @staticmethod def get_scheduler_jobs() -> list[SchedulerJobSchema]: jobs = SchedulerUtil.get_jobs() - return [ - SchedulerJobSchema( - id=job.id, - name=job.name, - trigger=str(job.trigger), - next_run_time=str(job.next_run_time) if job.next_run_time else None, - status=SchedulerUtil.get_job_status(job_id=job.id), + items: list[SchedulerJobSchema] = [] + for job in jobs: + schedulable, reason = SchedulerUtil.describe_schedulability(SchedulerUtil.job_code_block(job)) + items.append( + SchedulerJobSchema( + id=job.id, + name=job.name, + trigger=str(job.trigger), + # pending/未排程任务没有 next_run_time 属性（APScheduler 只在真正排程后设置），必须 getattr 兜底 + next_run_time=str(job.next_run_time) if getattr(job, "next_run_time", None) else None, + status=SchedulerUtil.get_job_status(job_id=job.id), + schedulable=schedulable, + unschedulable_reason=None if schedulable else reason, + ), ) - for job in jobs - ] + return items # ─── 调度器与任务运维（controller 唯一入口，禁止直调 SchedulerUtil）─── diff --git a/backend/app/modules/task/cronjob/node/schema.py b/backend/app/modules/task/cronjob/node/schema.py index 8d57600d..1e50e6f7 100644 --- a/backend/app/modules/task/cronjob/node/schema.py +++ b/backend/app/modules/task/cronjob/node/schema.py @@ -17,7 +17,14 @@ class NodeCreateSchema(BaseModel): """ name: str = Field(..., max_length=64, description="任务名称") - func: str | None = Field(default=None, description="代码块") + func: str | None = Field( + default=None, + description=( + "任务函数：两种写法——① 内置函数型 `builtin:<模块名>[.<函数名>]`（白名单解析后直接调用，不经 exec，" + "不受 SCHEDULER_ALLOW_CODE_EXEC 限制）；② 原始代码块（需定义 handler(*args, **kwargs)，等同服务器代码执行权限，" + "仅在 SCHEDULER_ALLOW_CODE_EXEC=true 时允许）" + ), + ) args: str | None = Field(default=None, description="位置参数") kwargs: str | None = Field(default=None, description="关键字参数") coalesce: bool | None = Field(default=False, description="是否合并运行:是否在多个运行时间到期时仅运行作业一次") @@ -102,6 +109,11 @@ class NodeOutSchema(NodeCreateSchema, BaseSchema, UserBySchema): next_run_time: str | None = Field(default=None, description="下次运行时间") last_run_time: str | None = Field(default=None, description="最近一次运行时间") last_run_status: int | None = Field(default=None, description="最近一次运行状态(2:成功 3:失败)") + schedulable: bool | None = Field( + default=None, + description="当前是否可调度；false 表示「已启用也不会执行」（如代码块被禁用、内置处理器不存在、未配置函数）", + ) + unschedulable_reason: str | None = Field(default=None, description="不可调度原因（含修复建议）") model_config = ConfigDict(from_attributes=True) diff --git a/backend/app/modules/task/cronjob/node/service.py b/backend/app/modules/task/cronjob/node/service.py index 9fe44a87..fadb53da 100644 --- a/backend/app/modules/task/cronjob/node/service.py +++ b/backend/app/modules/task/cronjob/node/service.py @@ -11,7 +11,6 @@ from croniter import croniter from sqlalchemy import false, select from sqlalchemy.ext.asyncio import AsyncSession -from app.config.setting import settings from app.core.ap_scheduler import SchedulerUtil, scheduler from app.core.base_schema import AuthSchema, PageResultSchema from app.core.exceptions import CustomException @@ -179,7 +178,17 @@ class NodeService: node_id = str(item.id) if item.id is not None else "" job = jobs.get(node_id) if job: - item.next_run_time = str(job.next_run_time) if job.next_run_time else None + item.next_run_time = str(job.next_run_time) if getattr(job, "next_run_time", None) else None + + # 可调度状态（t15）：区分「已启用且可执行」与「已启用但当前不会执行」， + # 并把后者以 WARN 打出（带节点标识），避免升级/开关调整后的静默失效 + schedulable, reason = SchedulerUtil.describe_schedulability(item.func) + item.schedulable = schedulable + item.unschedulable_reason = None if schedulable else reason + if not schedulable and item.status == 0: + _warn_unschedulable_node(node_id=node_id, name=item.name, reason=reason) + elif schedulable: + _clear_unschedulable_warning(node_id) stmt = ( select(JobModel) .where(JobModel.job_id == node_id, JobModel.is_deleted == false()) @@ -192,6 +201,41 @@ class NodeService: item.last_run_status = row.status +# 「已启用但不可调度」的告警去重：同一节点 + 同一原因只告警一次， +# 避免列表接口被反复调用时刷屏；节点恢复可调度时清除记录（见 _enrich_runtime）。 +_WARNED_UNSCHEDULABLE: dict[str, str] = {} + + +def _warn_unschedulable_node(*, node_id: str, name: str | None, reason: str | None) -> None: + """对「已启用但当前不可调度」的节点打一次 WARN（带节点标识与原因）。 + + 参数: + - node_id (str): 节点 ID（调度器 job id）。 + - name (str | None): 节点名称。 + - reason (str | None): 不可调度原因。 + + 返回: + - None + """ + text = reason or "" + if _WARNED_UNSCHEDULABLE.get(node_id) == text: + return + _WARNED_UNSCHEDULABLE[node_id] = text + logger.warning(f"节点任务不可调度（已启用但不会执行）: id={node_id} name={name!r} 原因={reason}") + + +def _clear_unschedulable_warning(node_id: str) -> None: + """节点恢复可调度后清除其告警去重记录（下次再失效仍会告警）。 + + 参数: + - node_id (str): 节点 ID。 + + 返回: + - None + """ + _WARNED_UNSCHEDULABLE.pop(node_id, None) + + # ── NodeModel 封装的调度操作（模块级函数，供 service / 启动自愈复用）──── @@ -281,8 +325,11 @@ def _add_job_with_trigger( code_block = job_info.func if not code_block or not code_block.strip(): raise ValueError("任务代码块不能为空") - if not settings.SCHEDULER_ALLOW_CODE_EXEC: - raise CustomException(msg="服务已禁用定时任务代码执行（SCHEDULER_ALLOW_CODE_EXEC=False），无法注册代码块任务") + # 可调度性判定（单一事实来源）：builtin: 引用走白名单、不经 exec，因此**开关关闭时也允许注册**； + # 只有"原始代码块"才会被 SCHEDULER_ALLOW_CODE_EXEC=False 拦住，并在提示里给出 builtin 替代写法。 + schedulable, reason = SchedulerUtil.describe_schedulability(code_block) + if not schedulable: + raise CustomException(msg=reason) jobstore = job_info.jobstore or "default" executor = job_info.executor or "threadpool" @@ -368,8 +415,10 @@ def run_node_once(node: NodeModel) -> str: """ if not node.func or not node.func.strip(): raise ValueError("任务代码块不能为空") - if not settings.SCHEDULER_ALLOW_CODE_EXEC: - raise CustomException(msg="服务已禁用定时任务代码执行（SCHEDULER_ALLOW_CODE_EXEC=False），无法执行代码块任务") + # 手动执行同样按可调度性判定：builtin 引用可直接执行；原始代码块在开关关闭时给出替代写法 + schedulable, reason = SchedulerUtil.describe_schedulability(node.func) + if not schedulable: + raise CustomException(msg=reason) temp_job_id = f"{node.id}{_MANUAL_JOB_PREFIX}{datetime.now():%Y%m%d%H%M%S}" trigger = DateTrigger(run_date=datetime.now() + timedelta(seconds=0.1), timezone="Asia/Shanghai") _add_job_with_trigger(node, trigger, job_id_override=temp_job_id, name_suffix=" - 手动执行") diff --git a/backend/app/modules/task/storage/core/base.py b/backend/app/modules/task/storage/core/base.py index 5e160023..ee85c51d 100644 --- a/backend/app/modules/task/storage/core/base.py +++ b/backend/app/modules/task/storage/core/base.py @@ -80,7 +80,7 @@ def decrypt_password(cipher: str | None) -> str: try: return CryptoUtil.decrypt(cipher) except CustomException as e: - raise CustomException(msg=f"存储源密码解密失败：{e!s}") + raise CustomException(msg=f"存储源密码解密失败：{e!s}", status_code=500) # 文件系统类协议与对象存储类协议的划分：仅用于文档说明与配置元数据， @@ -322,7 +322,7 @@ class BaseStorageAdapter(ABC): with ThreadPoolExecutor(max_workers=workers) as pool: list(pool.map(_run, tasks)) if errors: - raise CustomException(msg=f"批量上传失败 {len(errors)}/{len(tasks)} 个文件，首个错误: {errors[0]!s}") + raise CustomException(msg=f"批量上传失败 {len(errors)}/{len(tasks)} 个文件，首个错误: {errors[0]!s}", status_code=500) return len(tasks) def _sync_download_dir(self, remote_dir: str, local_dir: str, concurrency: int) -> int: @@ -352,7 +352,7 @@ class BaseStorageAdapter(ABC): with ThreadPoolExecutor(max_workers=workers) as pool: list(pool.map(_run, tasks)) if errors: - raise CustomException(msg=f"批量下载失败 {len(errors)}/{len(tasks)} 个文件，首个错误: {errors[0]!s}") + raise CustomException(msg=f"批量下载失败 {len(errors)}/{len(tasks)} 个文件，首个错误: {errors[0]!s}", status_code=500) return len(tasks) @staticmethod diff --git a/backend/app/modules/task/storage/core/cos_adapter.py b/backend/app/modules/task/storage/core/cos_adapter.py index c4cee05e..d669b8dd 100644 --- a/backend/app/modules/task/storage/core/cos_adapter.py +++ b/backend/app/modules/task/storage/core/cos_adapter.py @@ -96,7 +96,7 @@ class CosStorageAdapter(BaseStorageAdapter): EnableMD5=self._adv.enable_md5, ) except Exception as e: - raise CustomException(msg=f"COS 上传失败: {e!s}") + raise CustomException(msg=f"COS 上传失败: {e!s}", status_code=500) return remote_path def _sync_download(self, remote_path: str, local_path: str) -> str: @@ -107,7 +107,7 @@ class CosStorageAdapter(BaseStorageAdapter): DestFilePath=local_path, ) except Exception as e: - raise CustomException(msg=f"COS 下载失败: {e!s}") + raise CustomException(msg=f"COS 下载失败: {e!s}", status_code=500) return local_path def _sync_delete(self, remote_path: str) -> None: @@ -122,7 +122,7 @@ class CosStorageAdapter(BaseStorageAdapter): try: self.client.delete_object(Bucket=self._require_bucket(), Key=remote_path) except Exception as e: - raise CustomException(msg=f"COS 删除失败: {e!s}") + raise CustomException(msg=f"COS 删除失败: {e!s}", status_code=500) def _sync_exists(self, remote_path: str) -> bool: try: @@ -181,7 +181,7 @@ class CosStorageAdapter(BaseStorageAdapter): break return result except Exception as e: - raise CustomException(msg=f"COS 列表失败: {e!s}") + raise CustomException(msg=f"COS 列表失败: {e!s}", status_code=500) def _sync_list_page(self, prefix: str, page_size: int, cursor: str | None) -> StoragePage: """游标分页：单次 SDK 请求只拉一页（COS Marker），翻页经前端回传游标。""" @@ -250,7 +250,7 @@ class CosStorageAdapter(BaseStorageAdapter): next_cursor=next_cursor or None, ) except Exception as e: - raise CustomException(msg=f"COS 列表失败: {e!s}") + raise CustomException(msg=f"COS 列表失败: {e!s}", status_code=500) def _sync_list_buckets(self) -> list[str]: """列出账号下全部存储桶。""" @@ -259,7 +259,7 @@ class CosStorageAdapter(BaseStorageAdapter): buckets = (resp.get("Buckets") or {}).get("Bucket") or [] return [b.get("Name", "") for b in buckets if b.get("Name")] except Exception as e: - raise CustomException(msg=f"COS 桶列表失败: {e!s}") + raise CustomException(msg=f"COS 桶列表失败: {e!s}", status_code=500) def _sync_get_url(self, remote_path: str, expire: int) -> str: """生成预签名 URL @@ -280,7 +280,7 @@ class CosStorageAdapter(BaseStorageAdapter): Expired=expire, ) except Exception as e: - raise CustomException(msg=f"COS 生成预签名 URL 失败: {e!s}") + raise CustomException(msg=f"COS 生成预签名 URL 失败: {e!s}", status_code=500) # ── 目录操作（对象存储以 key/ 占位对象模拟目录）────────────────── @@ -311,13 +311,13 @@ class CosStorageAdapter(BaseStorageAdapter): try: self.client.put_object(Bucket=self._require_bucket(), Key=remote_dir.rstrip("/") + "/", Body=b"") except Exception as e: - raise CustomException(msg=f"COS 创建目录失败: {e!s}") + raise CustomException(msg=f"COS 创建目录失败: {e!s}", status_code=500) def _sync_rmdir(self, remote_dir: str) -> None: try: self.client.delete_object(Bucket=self._require_bucket(), Key=remote_dir.rstrip("/") + "/") except Exception as e: - raise CustomException(msg=f"COS 删除目录失败: {e!s}") + raise CustomException(msg=f"COS 删除目录失败: {e!s}", status_code=500) def _sync_copy(self, src: str, dst: str) -> None: """复制：目录走基类递归实现；文件用服务端 copy_object。""" @@ -331,7 +331,7 @@ class CosStorageAdapter(BaseStorageAdapter): CopySource={"Bucket": self._require_bucket(), "Key": src}, ) except Exception as e: - raise CustomException(msg=f"COS 复制失败: {e!s}") + raise CustomException(msg=f"COS 复制失败: {e!s}", status_code=500) def _sync_rename(self, src: str, dst: str) -> None: """重命名/移动：目录先复制后删除；文件 copy_object 后删源。""" @@ -345,7 +345,7 @@ class CosStorageAdapter(BaseStorageAdapter): try: keys = self._list_all_keys(prefix) except Exception as e: - raise CustomException(msg=f"COS 递归列表失败: {e!s}") + raise CustomException(msg=f"COS 递归列表失败: {e!s}", status_code=500) return self._entries_from_keys(keys) def _sync_delete_dir(self, remote_dir: str) -> None: @@ -361,4 +361,4 @@ class CosStorageAdapter(BaseStorageAdapter): Delete={"Objects": [{"Key": k} for k in keys[i : i + 1000]], "Quiet": True}, ) except Exception as e: - raise CustomException(msg=f"COS 递归删除失败: {e!s}") + raise CustomException(msg=f"COS 递归删除失败: {e!s}", status_code=500) diff --git a/backend/app/modules/task/storage/core/ftp_ftps_adapter.py b/backend/app/modules/task/storage/core/ftp_ftps_adapter.py index 91d17672..e061e116 100644 --- a/backend/app/modules/task/storage/core/ftp_ftps_adapter.py +++ b/backend/app/modules/task/storage/core/ftp_ftps_adapter.py @@ -101,7 +101,7 @@ class FtpStorageAdapter(BaseStorageAdapter): with open(local_path, "rb") as f: client.storbinary(f"STOR {remote_path}", f) except Exception as e: - raise CustomException(msg=f"FTP 上传失败: {e!s}") + raise CustomException(msg=f"FTP 上传失败: {e!s}", status_code=500) finally: self._close_client(client) return remote_path @@ -123,7 +123,7 @@ class FtpStorageAdapter(BaseStorageAdapter): with open(local_path, "wb") as f: client.retrbinary(f"RETR {remote_path}", f.write) except Exception as e: - raise CustomException(msg=f"FTP 下载失败: {e!s}") + raise CustomException(msg=f"FTP 下载失败: {e!s}", status_code=500) finally: self._close_client(client) return local_path @@ -133,7 +133,7 @@ class FtpStorageAdapter(BaseStorageAdapter): try: client.delete(remote_path) except Exception as e: - raise CustomException(msg=f"FTP 删除失败: {e!s}") + raise CustomException(msg=f"FTP 删除失败: {e!s}", status_code=500) finally: self._close_client(client) @@ -182,7 +182,7 @@ class FtpStorageAdapter(BaseStorageAdapter): ) return result except Exception as e: - raise CustomException(msg=f"FTP 列表失败: {e!s}") + raise CustomException(msg=f"FTP 列表失败: {e!s}", status_code=500) finally: self._close_client(client) @@ -200,7 +200,7 @@ class FtpStorageAdapter(BaseStorageAdapter): except ftplib.error_perm: pass # 目录已存在时跳过（部分服务器返回 550） except Exception as e: - raise CustomException(msg=f"FTP 创建目录失败: {e!s}") + raise CustomException(msg=f"FTP 创建目录失败: {e!s}", status_code=500) finally: self._close_client(client) @@ -209,7 +209,7 @@ class FtpStorageAdapter(BaseStorageAdapter): try: client.rmd(remote_dir) except Exception as e: - raise CustomException(msg=f"FTP 删除目录失败: {e!s}") + raise CustomException(msg=f"FTP 删除目录失败: {e!s}", status_code=500) finally: self._close_client(client) @@ -218,7 +218,7 @@ class FtpStorageAdapter(BaseStorageAdapter): try: client.rename(src, dst) except Exception as e: - raise CustomException(msg=f"FTP 重命名失败: {e!s}") + raise CustomException(msg=f"FTP 重命名失败: {e!s}", status_code=500) finally: self._close_client(client) @@ -248,6 +248,6 @@ class FtpStorageAdapter(BaseStorageAdapter): finally: os.remove(tmp_path) except Exception as e: - raise CustomException(msg=f"FTP 复制失败: {e!s}") + raise CustomException(msg=f"FTP 复制失败: {e!s}", status_code=500) finally: self._close_client(client) diff --git a/backend/app/modules/task/storage/core/local_adapter.py b/backend/app/modules/task/storage/core/local_adapter.py index 3f084d7b..6fa6a038 100644 --- a/backend/app/modules/task/storage/core/local_adapter.py +++ b/backend/app/modules/task/storage/core/local_adapter.py @@ -37,7 +37,7 @@ class LocalStorageAdapter(BaseStorageAdapter): os.makedirs(os.path.dirname(dest), exist_ok=True) shutil.copy2(local_path, dest) except Exception as e: - raise CustomException(msg=f"本地存储上传失败: {e!s}") + raise CustomException(msg=f"本地存储上传失败: {e!s}", status_code=500) return remote_path def _sync_download(self, remote_path: str, local_path: str) -> str: @@ -45,7 +45,7 @@ class LocalStorageAdapter(BaseStorageAdapter): try: shutil.copy2(src, local_path) except Exception as e: - raise CustomException(msg=f"本地存储下载失败: {e!s}") + raise CustomException(msg=f"本地存储下载失败: {e!s}", status_code=500) return local_path def _sync_delete(self, remote_path: str) -> None: @@ -56,7 +56,7 @@ class LocalStorageAdapter(BaseStorageAdapter): elif os.path.exists(target): os.remove(target) except Exception as e: - raise CustomException(msg=f"本地存储删除失败: {e!s}") + raise CustomException(msg=f"本地存储删除失败: {e!s}", status_code=500) def _sync_exists(self, remote_path: str) -> bool: return os.path.exists(self._abs_path(remote_path)) @@ -66,7 +66,7 @@ class LocalStorageAdapter(BaseStorageAdapter): try: entries = os.scandir(base_dir) except OSError as e: - raise CustomException(msg=f"本地存储列表失败: {e!s}") + raise CustomException(msg=f"本地存储列表失败: {e!s}", status_code=500) result: list[StorageObject] = [] for entry in entries: @@ -93,20 +93,20 @@ class LocalStorageAdapter(BaseStorageAdapter): try: os.makedirs(self._abs_path(remote_dir), exist_ok=True) except Exception as e: - raise CustomException(msg=f"本地存储创建目录失败: {e!s}") + raise CustomException(msg=f"本地存储创建目录失败: {e!s}", status_code=500) def _sync_rmdir(self, remote_dir: str) -> None: try: os.rmdir(self._abs_path(remote_dir)) except Exception as e: - raise CustomException(msg=f"本地存储删除目录失败: {e!s}") + raise CustomException(msg=f"本地存储删除目录失败: {e!s}", status_code=500) def _sync_rename(self, src: str, dst: str) -> None: try: os.makedirs(os.path.dirname(self._abs_path(dst)), exist_ok=True) shutil.move(self._abs_path(src), self._abs_path(dst)) except Exception as e: - raise CustomException(msg=f"本地存储重命名失败: {e!s}") + raise CustomException(msg=f"本地存储重命名失败: {e!s}", status_code=500) def _sync_copy(self, src: str, dst: str) -> None: src_abs, dst_abs = self._abs_path(src), self._abs_path(dst) @@ -117,4 +117,4 @@ class LocalStorageAdapter(BaseStorageAdapter): os.makedirs(os.path.dirname(dst_abs), exist_ok=True) shutil.copy2(src_abs, dst_abs) except Exception as e: - raise CustomException(msg=f"本地存储复制失败: {e!s}") + raise CustomException(msg=f"本地存储复制失败: {e!s}", status_code=500) diff --git a/backend/app/modules/task/storage/core/obs_adapter.py b/backend/app/modules/task/storage/core/obs_adapter.py index e3cd7dd1..56508789 100644 --- a/backend/app/modules/task/storage/core/obs_adapter.py +++ b/backend/app/modules/task/storage/core/obs_adapter.py @@ -176,7 +176,7 @@ class ObsStorageAdapter(BaseStorageAdapter): content=f, ) if not self._is_ok(resp): - raise CustomException(msg=f"OBS 上传失败: {self._error_desc(resp)}") + raise CustomException(msg=f"OBS 上传失败: {self._error_desc(resp)}", status_code=500) return remote_path part_size, concurrency, _ = self._multipart_settings() resp = self.client.uploadFile( @@ -188,11 +188,11 @@ class ObsStorageAdapter(BaseStorageAdapter): enableCheckpoint=True, ) if not self._is_ok(resp): - raise CustomException(msg=f"OBS 上传失败: {self._error_desc(resp)}") + raise CustomException(msg=f"OBS 上传失败: {self._error_desc(resp)}", status_code=500) except CustomException: raise except Exception as e: - raise CustomException(msg=f"OBS 上传失败: {e!s}") + raise CustomException(msg=f"OBS 上传失败: {e!s}", status_code=500) return remote_path def _sync_download(self, remote_path: str, local_path: str) -> str: @@ -207,22 +207,22 @@ class ObsStorageAdapter(BaseStorageAdapter): enableCheckpoint=True, ) if not self._is_ok(resp): - raise CustomException(msg=f"OBS 下载失败: {self._error_desc(resp)}") + raise CustomException(msg=f"OBS 下载失败: {self._error_desc(resp)}", status_code=500) except CustomException: raise except Exception as e: - raise CustomException(msg=f"OBS 下载失败: {e!s}") + raise CustomException(msg=f"OBS 下载失败: {e!s}", status_code=500) return local_path def _sync_delete(self, remote_path: str) -> None: try: resp = self.client.deleteObject(bucketName=self._require_bucket(), objectKey=remote_path) if not self._is_ok(resp): - raise CustomException(msg=f"OBS 删除失败: {self._error_desc(resp)}") + raise CustomException(msg=f"OBS 删除失败: {self._error_desc(resp)}", status_code=500) except CustomException: raise except Exception as e: - raise CustomException(msg=f"OBS 删除失败: {e!s}") + raise CustomException(msg=f"OBS 删除失败: {e!s}", status_code=500) def _sync_exists(self, remote_path: str) -> bool: try: @@ -255,7 +255,7 @@ class ObsStorageAdapter(BaseStorageAdapter): kwargs["marker"] = marker resp = self.client.listObjects(**kwargs) if not self._is_ok(resp): - raise CustomException(msg=f"OBS 列表失败: {self._error_desc(resp)}") + raise CustomException(msg=f"OBS 列表失败: {self._error_desc(resp)}", status_code=500) body = getattr(resp, "body", None) for common in getattr(body, "commonPrefixs", None) or []: raw_key = (getattr(common, "prefix", "") or "").rstrip("/") @@ -292,7 +292,7 @@ class ObsStorageAdapter(BaseStorageAdapter): except CustomException: raise except Exception as e: - raise CustomException(msg=f"OBS 列表失败: {e!s}") + raise CustomException(msg=f"OBS 列表失败: {e!s}", status_code=500) def _sync_list_page(self, prefix: str, page_size: int, cursor: str | None) -> StoragePage: """游标分页：单次 SDK 请求只拉一页（OBS marker），翻页经前端回传游标。""" @@ -307,7 +307,7 @@ class ObsStorageAdapter(BaseStorageAdapter): kwargs["marker"] = cursor resp = self.client.listObjects(**kwargs) if not self._is_ok(resp): - raise CustomException(msg=f"OBS 列表失败: {self._error_desc(resp)}") + raise CustomException(msg=f"OBS 列表失败: {self._error_desc(resp)}", status_code=500) body = getattr(resp, "body", None) truncated = bool(getattr(body, "is_truncated", None)) next_cursor = getattr(body, "next_marker", None) or None @@ -339,28 +339,28 @@ class ObsStorageAdapter(BaseStorageAdapter): except CustomException: raise except Exception as e: - raise CustomException(msg=f"OBS 列表失败: {e!s}") + raise CustomException(msg=f"OBS 列表失败: {e!s}", status_code=500) def _sync_list_buckets(self) -> list[str]: """列出账号下全部存储桶。""" try: resp = self.client.listBuckets() if not self._is_ok(resp): - raise CustomException(msg=f"OBS 桶列表失败: {self._error_desc(resp)}") + raise CustomException(msg=f"OBS 桶列表失败: {self._error_desc(resp)}", status_code=500) body = getattr(resp, "body", None) buckets = getattr(body, "buckets", None) or [] return [getattr(b, "name", "") or "" for b in buckets if getattr(b, "name", "")] except CustomException: raise except Exception as e: - raise CustomException(msg=f"OBS 桶列表失败: {e!s}") + raise CustomException(msg=f"OBS 桶列表失败: {e!s}", status_code=500) def _sync_get_url(self, remote_path: str, expire: int) -> str: try: resp = self.client.createSignedUrl("GET", bucketName=self._require_bucket(), objectKey=remote_path, expires=expire) return getattr(resp, "signedUrl", "") except Exception as e: - raise CustomException(msg=f"OBS 生成预签名 URL 失败: {e!s}") + raise CustomException(msg=f"OBS 生成预签名 URL 失败: {e!s}", status_code=500) # ── 目录操作（对象存储以 key/ 占位对象模拟目录）────────────────── @@ -374,7 +374,7 @@ class ObsStorageAdapter(BaseStorageAdapter): kwargs["marker"] = marker resp = self.client.listObjects(**kwargs) if not self._is_ok(resp): - raise CustomException(msg=f"OBS 列举对象失败: {self._error_desc(resp)}") + raise CustomException(msg=f"OBS 列举对象失败: {self._error_desc(resp)}", status_code=500) body = getattr(resp, "body", None) for obj in getattr(body, "contents", None) or []: key = getattr(obj, "key", "") @@ -403,21 +403,21 @@ class ObsStorageAdapter(BaseStorageAdapter): try: resp = self.client.putContent(bucketName=self._require_bucket(), objectKey=remote_dir.rstrip("/") + "/", content=b"") if not self._is_ok(resp): - raise CustomException(msg=f"OBS 创建目录失败: {self._error_desc(resp)}") + raise CustomException(msg=f"OBS 创建目录失败: {self._error_desc(resp)}", status_code=500) except CustomException: raise except Exception as e: - raise CustomException(msg=f"OBS 创建目录失败: {e!s}") + raise CustomException(msg=f"OBS 创建目录失败: {e!s}", status_code=500) def _sync_rmdir(self, remote_dir: str) -> None: try: resp = self.client.deleteObject(bucketName=self._require_bucket(), objectKey=remote_dir.rstrip("/") + "/") if not self._is_ok(resp): - raise CustomException(msg=f"OBS 删除目录失败: {self._error_desc(resp)}") + raise CustomException(msg=f"OBS 删除目录失败: {self._error_desc(resp)}", status_code=500) except CustomException: raise except Exception as e: - raise CustomException(msg=f"OBS 删除目录失败: {e!s}") + raise CustomException(msg=f"OBS 删除目录失败: {e!s}", status_code=500) def _sync_copy(self, src: str, dst: str) -> None: """复制：目录走基类递归实现；文件用服务端 copyObject。""" @@ -432,11 +432,11 @@ class ObsStorageAdapter(BaseStorageAdapter): destObjectKey=dst, ) if not self._is_ok(resp): - raise CustomException(msg=f"OBS 复制失败: {self._error_desc(resp)}") + raise CustomException(msg=f"OBS 复制失败: {self._error_desc(resp)}", status_code=500) except CustomException: raise except Exception as e: - raise CustomException(msg=f"OBS 复制失败: {e!s}") + raise CustomException(msg=f"OBS 复制失败: {e!s}", status_code=500) def _sync_rename(self, src: str, dst: str) -> None: """重命名/移动：优先 OBS 原生 renameFile（支持目录级），失败回退先复制后删除。""" @@ -456,7 +456,7 @@ class ObsStorageAdapter(BaseStorageAdapter): try: keys = self._list_all_keys(prefix) except Exception as e: - raise CustomException(msg=f"OBS 递归列表失败: {e!s}") + raise CustomException(msg=f"OBS 递归列表失败: {e!s}", status_code=500) return self._entries_from_keys(keys) def _sync_delete_dir(self, remote_dir: str) -> None: @@ -473,11 +473,11 @@ class ObsStorageAdapter(BaseStorageAdapter): deleteObjectsRequest={"quiet": True, "objects": [{"key": k} for k in batch]}, ) if not self._is_ok(resp): - raise CustomException(msg=f"OBS 递归删除失败: {self._error_desc(resp)}") + raise CustomException(msg=f"OBS 递归删除失败: {self._error_desc(resp)}", status_code=500) except CustomException: raise except Exception as e: - raise CustomException(msg=f"OBS 递归删除失败: {e!s}") + raise CustomException(msg=f"OBS 递归删除失败: {e!s}", status_code=500) def _sync_close(self) -> None: """关闭 OBS 客户端连接。""" diff --git a/backend/app/modules/task/storage/core/oss_adapter.py b/backend/app/modules/task/storage/core/oss_adapter.py index fa7c105e..9a12eff8 100644 --- a/backend/app/modules/task/storage/core/oss_adapter.py +++ b/backend/app/modules/task/storage/core/oss_adapter.py @@ -74,7 +74,7 @@ class OssStorageAdapter(BaseStorageAdapter): parallel_num=concurrency, ) except Exception as e: - raise CustomException(msg=f"OSS 上传失败: {e!s}") + raise CustomException(msg=f"OSS 上传失败: {e!s}", status_code=500) return remote_path def _sync_download(self, remote_path: str, local_path: str) -> str: @@ -84,14 +84,14 @@ class OssStorageAdapter(BaseStorageAdapter): local_path, ) except Exception as e: - raise CustomException(msg=f"OSS 下载失败: {e!s}") + raise CustomException(msg=f"OSS 下载失败: {e!s}", status_code=500) return local_path def _sync_delete(self, remote_path: str) -> None: try: self.client.delete_object(oss.DeleteObjectRequest(bucket=self.bucket_name, key=remote_path)) except Exception as e: - raise CustomException(msg=f"OSS 删除失败: {e!s}") + raise CustomException(msg=f"OSS 删除失败: {e!s}", status_code=500) def _sync_exists(self, remote_path: str) -> bool: try: @@ -140,7 +140,7 @@ class OssStorageAdapter(BaseStorageAdapter): ) return result except Exception as e: - raise CustomException(msg=f"OSS 列表失败: {e!s}") + raise CustomException(msg=f"OSS 列表失败: {e!s}", status_code=500) def _sync_list_page(self, prefix: str, page_size: int, cursor: str | None) -> StoragePage: """游标分页：单次 SDK 请求只拉一页（OSS continuation_token），翻页经前端回传游标。""" @@ -178,7 +178,7 @@ class OssStorageAdapter(BaseStorageAdapter): ) return StoragePage(items=items, has_next=truncated, next_cursor=next_cursor) except Exception as e: - raise CustomException(msg=f"OSS 列表失败: {e!s}") + raise CustomException(msg=f"OSS 列表失败: {e!s}", status_code=500) def _sync_list_buckets(self) -> list[str]: """列出账号下全部存储桶。""" @@ -191,7 +191,7 @@ class OssStorageAdapter(BaseStorageAdapter): result.append(b.name) return result except Exception as e: - raise CustomException(msg=f"OSS 桶列表失败: {e!s}") + raise CustomException(msg=f"OSS 桶列表失败: {e!s}", status_code=500) def _sync_get_url(self, remote_path: str, expire: int) -> str | None: try: @@ -201,7 +201,7 @@ class OssStorageAdapter(BaseStorageAdapter): ) return result.url except Exception as e: - raise CustomException(msg=f"OSS 生成预签名 URL 失败: {e!s}") + raise CustomException(msg=f"OSS 生成预签名 URL 失败: {e!s}", status_code=500) # ── 目录操作（对象存储以 key/ 占位对象模拟目录）────────────────── @@ -231,13 +231,13 @@ class OssStorageAdapter(BaseStorageAdapter): try: self.client.put_object(oss.PutObjectRequest(bucket=self.bucket_name, key=remote_dir.rstrip("/") + "/", body=b"")) except Exception as e: - raise CustomException(msg=f"OSS 创建目录失败: {e!s}") + raise CustomException(msg=f"OSS 创建目录失败: {e!s}", status_code=500) def _sync_rmdir(self, remote_dir: str) -> None: try: self.client.delete_object(oss.DeleteObjectRequest(bucket=self.bucket_name, key=remote_dir.rstrip("/") + "/")) except Exception as e: - raise CustomException(msg=f"OSS 删除目录失败: {e!s}") + raise CustomException(msg=f"OSS 删除目录失败: {e!s}", status_code=500) def _sync_copy(self, src: str, dst: str) -> None: """复制：目录走基类递归实现；文件用服务端 copy_object。""" @@ -249,7 +249,7 @@ class OssStorageAdapter(BaseStorageAdapter): oss.CopyObjectRequest(bucket=self.bucket_name, key=dst, source_bucket=self.bucket_name, source_key=src) ) except Exception as e: - raise CustomException(msg=f"OSS 复制失败: {e!s}") + raise CustomException(msg=f"OSS 复制失败: {e!s}", status_code=500) def _sync_rename(self, src: str, dst: str) -> None: """重命名/移动：目录先复制后删除；文件 copy_object 后删源。""" @@ -263,7 +263,7 @@ class OssStorageAdapter(BaseStorageAdapter): try: keys = self._list_all_keys(prefix) except Exception as e: - raise CustomException(msg=f"OSS 递归列表失败: {e!s}") + raise CustomException(msg=f"OSS 递归列表失败: {e!s}", status_code=500) return self._entries_from_keys(keys) def _sync_delete_dir(self, remote_dir: str) -> None: @@ -282,4 +282,4 @@ class OssStorageAdapter(BaseStorageAdapter): ) ) except Exception as e: - raise CustomException(msg=f"OSS 递归删除失败: {e!s}") + raise CustomException(msg=f"OSS 递归删除失败: {e!s}", status_code=500) diff --git a/backend/app/modules/task/storage/core/s3_adapter.py b/backend/app/modules/task/storage/core/s3_adapter.py index 5760f05b..11f759c4 100644 --- a/backend/app/modules/task/storage/core/s3_adapter.py +++ b/backend/app/modules/task/storage/core/s3_adapter.py @@ -88,21 +88,21 @@ class S3StorageAdapter(BaseStorageAdapter): ) self.client.upload_file(local_path, self._require_bucket(), remote_path, Config=transfer_config) except Exception as e: - raise CustomException(msg=f"S3 上传失败: {e!s}") + raise CustomException(msg=f"S3 上传失败: {e!s}", status_code=500) return remote_path def _sync_download(self, remote_path: str, local_path: str) -> str: try: self.client.download_file(self._require_bucket(), remote_path, local_path) except Exception as e: - raise CustomException(msg=f"S3 下载失败: {e!s}") + raise CustomException(msg=f"S3 下载失败: {e!s}", status_code=500) return local_path def _sync_delete(self, remote_path: str) -> None: try: self.client.delete_object(Bucket=self._require_bucket(), Key=remote_path) except Exception as e: - raise CustomException(msg=f"S3 删除失败: {e!s}") + raise CustomException(msg=f"S3 删除失败: {e!s}", status_code=500) def _sync_exists(self, remote_path: str) -> bool: try: @@ -163,7 +163,7 @@ class S3StorageAdapter(BaseStorageAdapter): break return result except Exception as e: - raise CustomException(msg=f"S3 列表失败: {e!s}") + raise CustomException(msg=f"S3 列表失败: {e!s}", status_code=500) def _sync_list_page(self, prefix: str, page_size: int, cursor: str | None) -> StoragePage: """游标分页：单次 SDK 请求只拉一页（S3 ContinuationToken），翻页经前端回传游标。""" @@ -203,7 +203,7 @@ class S3StorageAdapter(BaseStorageAdapter): next_cursor=resp.get("NextContinuationToken"), ) except Exception as e: - raise CustomException(msg=f"S3 列表失败: {e!s}") + raise CustomException(msg=f"S3 列表失败: {e!s}", status_code=500) def _sync_list_buckets(self) -> list[str]: """列出账号下全部存储桶。""" @@ -211,7 +211,7 @@ class S3StorageAdapter(BaseStorageAdapter): resp = self.client.list_buckets() return [b.get("Name", "") for b in resp.get("Buckets", []) if b.get("Name")] except Exception as e: - raise CustomException(msg=f"S3 桶列表失败: {e!s}") + raise CustomException(msg=f"S3 桶列表失败: {e!s}", status_code=500) def _sync_get_url(self, remote_path: str, expire: int) -> str: try: @@ -221,7 +221,7 @@ class S3StorageAdapter(BaseStorageAdapter): ExpiresIn=expire, ) except Exception as e: - raise CustomException(msg=f"S3 生成预签名 URL 失败: {e!s}") + raise CustomException(msg=f"S3 生成预签名 URL 失败: {e!s}", status_code=500) def _sync_close(self) -> None: """关闭 boto3 客户端连接。""" @@ -253,13 +253,13 @@ class S3StorageAdapter(BaseStorageAdapter): try: self.client.put_object(Bucket=self._require_bucket(), Key=remote_dir.rstrip("/") + "/", Body=b"") except Exception as e: - raise CustomException(msg=f"S3 创建目录失败: {e!s}") + raise CustomException(msg=f"S3 创建目录失败: {e!s}", status_code=500) def _sync_rmdir(self, remote_dir: str) -> None: try: self.client.delete_object(Bucket=self._require_bucket(), Key=remote_dir.rstrip("/") + "/") except Exception as e: - raise CustomException(msg=f"S3 删除目录失败: {e!s}") + raise CustomException(msg=f"S3 删除目录失败: {e!s}", status_code=500) def _sync_copy(self, src: str, dst: str) -> None: """复制：目录走基类递归实现；文件用服务端 copy_object。""" @@ -273,7 +273,7 @@ class S3StorageAdapter(BaseStorageAdapter): Key=dst, ) except Exception as e: - raise CustomException(msg=f"S3 复制失败: {e!s}") + raise CustomException(msg=f"S3 复制失败: {e!s}", status_code=500) def _sync_rename(self, src: str, dst: str) -> None: """重命名/移动：目录先复制后删除；文件 copy_object 后删源。""" @@ -287,7 +287,7 @@ class S3StorageAdapter(BaseStorageAdapter): try: keys = self._list_all_keys(prefix) except Exception as e: - raise CustomException(msg=f"S3 递归列表失败: {e!s}") + raise CustomException(msg=f"S3 递归列表失败: {e!s}", status_code=500) return self._entries_from_keys(keys) def _sync_delete_dir(self, remote_dir: str) -> None: @@ -303,4 +303,4 @@ class S3StorageAdapter(BaseStorageAdapter): Delete={"Objects": [{"Key": k} for k in keys[i : i + 1000]], "Quiet": True}, ) except Exception as e: - raise CustomException(msg=f"S3 递归删除失败: {e!s}") + raise CustomException(msg=f"S3 递归删除失败: {e!s}", status_code=500) diff --git a/backend/app/modules/task/storage/core/sftp_adapter.py b/backend/app/modules/task/storage/core/sftp_adapter.py index d9315a1c..f1612473 100644 --- a/backend/app/modules/task/storage/core/sftp_adapter.py +++ b/backend/app/modules/task/storage/core/sftp_adapter.py @@ -118,7 +118,7 @@ class SftpStorageAdapter(BaseStorageAdapter): self._ensure_remote_dir(client, dir_part) client.put(local_path, remote_path) except Exception as e: - raise CustomException(msg=f"SFTP 上传失败: {e!s}") + raise CustomException(msg=f"SFTP 上传失败: {e!s}", status_code=500) return remote_path def _sync_download(self, remote_path: str, local_path: str) -> str: @@ -141,14 +141,14 @@ class SftpStorageAdapter(BaseStorageAdapter): else: client.get(remote_path, local_path) except Exception as e: - raise CustomException(msg=f"SFTP 下载失败: {e!s}") + raise CustomException(msg=f"SFTP 下载失败: {e!s}", status_code=500) return local_path def _sync_delete(self, remote_path: str) -> None: try: self._connect().remove(remote_path) except Exception as e: - raise CustomException(msg=f"SFTP 删除失败: {e!s}") + raise CustomException(msg=f"SFTP 删除失败: {e!s}", status_code=500) def _sync_exists(self, remote_path: str) -> bool: try: @@ -162,7 +162,7 @@ class SftpStorageAdapter(BaseStorageAdapter): # 空 prefix 表示浏览根目录：SFTP 用 "." 表示登录后的当前目录 attrs = self._connect().listdir_attr(prefix or ".") except Exception as e: - raise CustomException(msg=f"SFTP 列表失败: {e!s}") + raise CustomException(msg=f"SFTP 列表失败: {e!s}", status_code=500) result: list[StorageObject] = [] for attr in attrs: result.append( @@ -189,19 +189,19 @@ class SftpStorageAdapter(BaseStorageAdapter): try: self._ensure_remote_dir(self._connect(), remote_dir) except Exception as e: - raise CustomException(msg=f"SFTP 创建目录失败: {e!s}") + raise CustomException(msg=f"SFTP 创建目录失败: {e!s}", status_code=500) def _sync_rmdir(self, remote_dir: str) -> None: try: self._connect().rmdir(remote_dir) except Exception as e: - raise CustomException(msg=f"SFTP 删除目录失败: {e!s}") + raise CustomException(msg=f"SFTP 删除目录失败: {e!s}", status_code=500) def _sync_rename(self, src: str, dst: str) -> None: try: self._connect().posix_rename(src, dst) except Exception as e: - raise CustomException(msg=f"SFTP 重命名失败: {e!s}") + raise CustomException(msg=f"SFTP 重命名失败: {e!s}", status_code=500) def _sync_copy(self, src: str, dst: str) -> None: """复制：目录走基类递归实现；文件用通道流式复制（SFTP 无服务端复制）。""" @@ -214,4 +214,4 @@ class SftpStorageAdapter(BaseStorageAdapter): while chunk := rf.read(1024 * 1024): wf.write(chunk) except Exception as e: - raise CustomException(msg=f"SFTP 复制失败: {e!s}") + raise CustomException(msg=f"SFTP 复制失败: {e!s}", status_code=500) diff --git a/backend/app/plugin/module_example/demo/service.py b/backend/app/plugin/module_example/demo/service.py index b178f0e8..90b8896d 100644 --- a/backend/app/plugin/module_example/demo/service.py +++ b/backend/app/plugin/module_example/demo/service.py @@ -262,7 +262,7 @@ class DemoService: except Exception as e: logger.error(f"批量导入示例失败: {e!s}") - raise CustomException(msg=f"导入失败: {e!s}") from e + raise CustomException(msg=f"导入失败: {e!s}", status_code=500) from e @staticmethod def import_template_download() -> bytes: diff --git a/backend/app/utils/crypto_util.py b/backend/app/utils/crypto_util.py index 346fe147..f8f54471 100644 --- a/backend/app/utils/crypto_util.py +++ b/backend/app/utils/crypto_util.py @@ -71,7 +71,7 @@ class CryptoUtil: return fernet.decrypt(cipher.encode("utf-8")).decode("utf-8") except (InvalidToken, ValueError): continue - raise CustomException(msg="敏感数据解密失败，可能原因：DATA_ENCRYPTION_KEY/SECRET_KEY 变更或数据损坏") + raise CustomException(msg="敏感数据解密失败，可能原因：DATA_ENCRYPTION_KEY/SECRET_KEY 变更或数据损坏", status_code=500) @classmethod def decrypt_or_keep(cls, value: str | None) -> str: diff --git a/backend/app/utils/import_util.py b/backend/app/utils/import_util.py index bea2f914..c993db0e 100644 --- a/backend/app/utils/import_util.py +++ b/backend/app/utils/import_util.py @@ -154,7 +154,7 @@ class ImportUtil: if "cannot import name" not in str(e): raise ImportError(f"❗️ 警告: 无法导入模块 {module_name}: {e}") except Exception as e: - raise CustomException(f"❌️ 处理模块 {module_name} 时出错: {e}") + raise CustomException(f"❌️ 处理模块 {module_name} 时出错: {e}", status_code=500) # 查找apscheduler_jobs表的模型（如果存在） cls._find_apscheduler_model(base_class, models, seen_models, seen_tables) @@ -205,4 +205,4 @@ class ImportUtil: except ImportError: pass except Exception as e: - raise CustomException(f"❗️ 查找APScheduler模型时出错: {e}") + raise CustomException(f"❗️ 查找APScheduler模型时出错: {e}", status_code=500) diff --git a/backend/app/utils/upload_util.py b/backend/app/utils/upload_util.py index 81626ab4..4e4fb156 100644 --- a/backend/app/utils/upload_util.py +++ b/backend/app/utils/upload_util.py @@ -468,7 +468,7 @@ class UploadUtil: raise except Exception as e: logger.error(f"文件上传失败: {e}") - raise CustomException(msg=f"文件上传失败: {e}") + raise CustomException(msg=f"文件上传失败: {e}", status_code=500) @staticmethod def get_file_tree(file_path: str) -> list[dict]: diff --git a/backend/env/.env.example b/backend/env/.env.example index 7126b90e..7ebf2212 100644 --- a/backend/env/.env.example +++ b/backend/env/.env.example @@ -35,7 +35,66 @@ REDIS_DB_NAME = 1 # 日志级别配置 LOGGER_LEVEL = DEBUG +# ================================================= # +# *************** 安全相关（生产必填）*************** # +# ================================================= # +# 令牌签名密钥：openssl rand -hex 32（更换后所有已签发令牌失效） +SECRET_KEY = change_me_generate_with_openssl_rand_hex_32 +# 落库敏感数据加密主密钥：openssl rand -hex 32；留空则由 SECRET_KEY 派生 +DATA_ENCRYPTION_KEY = '' +# 轮换前的旧主密钥（逗号分隔），仅用于解密历史密文 +DATA_ENCRYPTION_OLD_KEYS = '' +# 生产跨域白名单（逗号分隔，如 https://admin.example.com,https://www.example.com） +# ⚠️ 生产留空 = 拒绝全部跨域请求（安全默认，不再回落通配 "*"；同源部署不受影响）。 +# 存在站外集成时必须显式列出域名；填 "*" 会被忽略（与 ALLOW_CREDENTIALS=True 组合不可接受）。 +PROD_CORS_ORIGINS = '' +# 严格启动：true 时「生产未配置 PROD_CORS_ORIGINS」直接启动失败（默认 false 只打 warning） +CORS_STRICT_STARTUP = False +# 只信任该网段内的反向代理传来的 X-Forwarded-*（切勿使用 *） +FORWARDED_ALLOW_IPS = 10.255.178.0/24 +# 演示模式：true 时禁止写操作 +DEMO_ENABLE = False + +# ================================================= # +# ******************* 定时任务 ******************* # +# ================================================= # +# 是否允许定时任务执行用户提交的代码块（exec，等同服务器代码执行权限）。 +# ⚠️ 默认 False：调成 false（或保持默认）时，**所有「代码块型」节点任务都不会被调度** +# （注册/手动执行会返回明确错误："服务已禁用定时任务代码执行…"），执行结果可在任务执行日志查看； +# 这只关闭 exec 能力，不影响调度器本身、也不影响无代码块的其它任务。 +# 如确需该能力，请显式置为 true，并按最小权限授予任务管理权限。 +SCHEDULER_ALLOW_CODE_EXEC = False + +# ================================================= # +# ******************* 主机名校验 ******************* # +# ================================================= # +# 允许的 Host 头（逗号分隔的 JSON 数组）；生产填真实域名 +# ALLOWED_HOSTS = ["service.fastapiadmin.com", "*.fastapiadmin.com"] +# 容器内健康检查/本地直连需要 Host: localhost|127.0.0.1|[::1]；生产不需要时置 False 收敛校验范围 +ALLOW_LOCALHOST_HOSTS = True +# OAuth 回调域名白名单（**不再是通配符 "*"**）；非生产默认额外放行本机与回环地址 +# OAUTH_ALLOWED_HOSTS = ["service.fastapiadmin.com", "*.fastapiadmin.com", "localhost", "127.0.0.1", "[::1]"] + # 大模型配置 OPENAI_BASE_URL = https://api.deepseek.com OPENAI_API_KEY = your_api_key OPENAI_MODEL = deepseek-v4-flash + +# ================================================= # +# *********** 升级说明（行为变更，请逐条确认）*********** +# ================================================= # +# 1) SCHEDULER_ALLOW_CODE_EXEC 默认由 true 改为 false：升级后所有「代码块型」节点任务 +# 不再被调度/执行，直至显式配置为 true。健康检查（/api/v1/monitor/health/check）新增 +# code_exec_enabled 字段可用于确认当前生效值。 +# 2) 生产 PROD_CORS_ORIGINS 未配置时不再回落通配 "*"，而是拒绝全部跨域请求 +# （同源部署不受影响）。健康检查新增 cors_origins_configured 字段；可用 CORS_STRICT_STARTUP=true +# 把它变成启动失败以便尽早发现。 +# 3) 业务异常的 HTTP 状态码按语义返回：客户端可修正类为 4xx（参数/验证码 400、无权限 403、 +# 资源不存在 404），包装底层异常的内部故障为 500（此前统一 500/后统一 400 均不正确）。 +# 4) 代码生成器的列名必须是合法标识符（且不能是 Python 关键字 / JS 保留字）， +# 列类型必须是「类型名 + 可选(长度[,精度]) + 可选[] + 可选 unsigned」的形状； +# 含引号/分号/任意自由文本的类型会被拒绝（ENUM/SET 会归一化为基名）。 +# 5) 代码生成落盘前新增结构闸门：.py 产物必须与「哨兵值骨架」AST 一致，.vue/.ts/.toml 做行结构比对， +# 任何注入/转义缺口都会在写盘前报错，而不是写出可被启动期 import 的坏文件。 +# 6) OAUTH_ALLOWED_HOSTS 默认不再是 "*"；如需自定义请在对应环境中显式配置。 +# 7) ALLOWED_HOSTS 默认不再无条件包含 localhost：由 ALLOW_LOCALHOST_HOSTS 控制（默认 True 以兼容容器健康检查）。 diff --git a/backend/templates/vue/index.vue.jinja2 b/backend/templates/vue/index.vue.jinja2 index 8c987648..8cbe0b25 100644 --- a/backend/templates/vue/index.vue.jinja2 +++ b/backend/templates/vue/index.vue.jinja2 @@ -111,18 +111,18 @@ class="crud-dialog-art-form" /> {% if table.sub %} - {{ table.sub_table.function_name }}列表 + {{ table.sub_table.function_name | vue_label }}列表
- 新增{{ table.sub_table.function_name }} + 新增{{ table.sub_table.function_name | vue_label }}
{% for col in table.sub_table.columns %} {% if col.is_list and col.column_name != table.sub_table.pk_column.column_name %} {% set col_comment = col.column_comment if col.column_comment else col.column_name %} - + {% endif %} {% endfor %} @@ -141,7 +141,7 @@ diff --git a/backend/tests/test_generator_template_safety.py b/backend/tests/test_generator_template_safety.py new file mode 100644 index 00000000..5dcd1945 --- /dev/null +++ b/backend/tests/test_generator_template_safety.py @@ -0,0 +1,546 @@ +"""代码生成器模板安全（注入面）回归测试。 + +背景：``backend/audit-backend.md`` B1 —— 代码生成模板以 ``autoescape=False`` 渲染，并把 +``function_name`` / ``column_comment`` / ``column_name`` 原样写进 ``.py``，产物落在 +``backend/app/plugin/**`` 且在应用启动时被 ``app/core/discover.py`` import，等价于「有代码生成 +权限 = 服务器任意代码执行」。 + +本测试固化两条不变量： +1. 恶意输入（引号、换行、闭合序列、内嵌 import/exec）不能变成**可执行代码**； +2. 恶意输入之下，渲染产物**仍是语法合法的 Python**（且正常文本按原义保留）。 +""" + +import ast +from typing import Any + +import pytest +from pydantic import ValidationError + +from app.modules.generator.gencode.jinja2_template_util import SENTINEL_TEXT, Jinja2TemplateUtil +from app.modules.generator.gencode.schema import GenTableColumnOutSchema, GenTableOutSchema +from app.modules.generator.gencode.template_safety import ( + assert_python_structure, + assert_python_syntax, + assert_text_structure, + escape_template_text, + html_attr, + is_safe_identifier, + reserved_word_reason, + safe_ident, + unescape_template_text, + vue_label, +) + +# ── 恶意载荷：覆盖「引号 / 换行 / 闭合序列 / 内嵌可执行语句」四类 ────────────── +EVIL_COMMENT = "'); import os; os.system('id'); #" +EVIL_FUNCTION_NAME = 'x\n"""\nimport os\nos.system("id")\n"""\n' +EVIL_COLUMN_NAME = 'evil"); import os; os.system("id"); #' +EVIL_BACKTICK = "`${alert(1)}`" + + +def _column(**overrides: Any) -> GenTableColumnOutSchema: + """构造一个字段模型（默认是普通 varchar 列）。""" + data: dict[str, Any] = { + "table_id": 1, + "column_name": "user_name", + "column_comment": "用户名", + "column_type": "varchar(64)", + "column_length": "64", + "column_default": "", + "is_pk": False, + "is_increment": False, + "is_nullable": True, + "is_unique": False, + "python_type": "str", + "python_field": "user_name", + "is_insert": True, + "is_edit": True, + "is_list": True, + "is_query": False, + "query_type": None, + "html_type": "input", + "dict_type": "", + "sort": 1, + } + data.update(overrides) + return GenTableColumnOutSchema(**data) + + +def _table(columns: list[GenTableColumnOutSchema] | None = None, **overrides: Any) -> GenTableOutSchema: + """构造一个输出模型（默认包名/模块名已是工程约定形态）。""" + cols = columns if columns is not None else [_column()] + pk = next((c for c in cols if c.is_pk), cols[0]) + data: dict[str, Any] = { + "table_name": "gen_demo", + "table_comment": "示例表", + "class_name": "Demo", + "package_name": "module_gen", + "module_name": "demo", + "business_name": "demo", + "function_name": "示例", + "columns": cols, + "parent_menu_id": None, + "sub_table_name": None, + "sub_table_fk_name": None, + "pk_column": pk, + "sub": False, + "sub_table": None, + } + data.update(overrides) + return GenTableOutSchema(**data) + + +async def _render_all(table: GenTableOutSchema) -> dict[str, str]: + """按生产路径（prepare_context + 全部模板）渲染所有产物。""" + env = Jinja2TemplateUtil.get_env() + context = Jinja2TemplateUtil.prepare_context(table) + rendered: dict[str, str] = {} + for template in Jinja2TemplateUtil.get_template_list(): + rendered[Jinja2TemplateUtil.get_file_name(template, table)] = await env.get_template(template).render_async(**context) + return rendered + + +def _python_outputs(rendered: dict[str, str]) -> dict[str, str]: + """只取 .py 产物。""" + return {name: content for name, content in rendered.items() if name.endswith(".py")} + + +def _dangerous_ast_nodes(source: str) -> list[str]: + """返回源码中「可执行注入」的 AST 证据（import os / os.system / exec）。""" + tree = ast.parse(source) + findings: list[str] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + findings.extend(f"import {alias.name}" for alias in node.names if alias.name.split(".")[0] == "os") + elif isinstance(node, ast.ImportFrom) and (node.module or "").split(".")[0] == "os": + findings.append(f"from {node.module} import") + elif isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr in {"system", "popen", "exec", "eval"}: + findings.append(f".{node.func.attr}() 调用") + return findings + + +# ── 1. 纯函数级 ──────────────────────────────────────────────────────────── + + +def test_escape_template_text_neutralizes_quote_and_newline() -> None: + """转义后不再含可闭合字面量/可换行的字符，且保留原字符的语义（\\uXXXX）。""" + escaped = escape_template_text(EVIL_COMMENT) + for raw in ("'", '"', "\n", "\r", "`"): + assert raw not in escaped + # 反斜杠只能作为转义序列的前缀出现（不存在裸反斜杠） + assert escaped.count("\\") == escaped.count("\\u") + assert "\\u0027" in escaped # 单引号被转义 + assert "os.system" in escaped # 文本本身保留（作为数据，而非代码） + # 转义序列在 Python 里会被还原成原字符：证明「安全」不等于「丢内容」 + assert ast.literal_eval(f"'{escaped}'") == EVIL_COMMENT + + +def test_escape_template_text_breaks_js_interpolation() -> None: + """${...} 会被打断，避免在 JS 模板字符串里变成插值表达式。""" + escaped = escape_template_text(EVIL_BACKTICK) + assert "${" not in escaped + assert "\\u0060" in escaped + + +def test_safe_ident_normalizes_and_preserves_legal_names() -> None: + """合法标识符原样保留；非法字符被替换；数字开头补下划线；空值取默认。""" + assert safe_ident("user_name") == "user_name" + assert safe_ident("_private") == "_private" + assert safe_ident("a__b") == "a__b" + assert safe_ident("2fa") == "_2fa" + assert safe_ident("") == "entity" + assert safe_ident(None, default="") == "" + # 注入型列名被规范为合法标识符（无法再闭合并列语句） + mangled = safe_ident(EVIL_COLUMN_NAME) + assert is_safe_identifier(mangled) + assert not any(ch in mangled for ch in ("'", '"', " ", ";", "(", ")")) + + +def test_is_safe_identifier() -> None: + """白名单判定：合法/非法/超长/空值。""" + assert is_safe_identifier("user_name") + assert is_safe_identifier("_x1") + assert not is_safe_identifier("2fa") + assert not is_safe_identifier("a b") + assert not is_safe_identifier(EVIL_COLUMN_NAME) + assert not is_safe_identifier("") + assert not is_safe_identifier("a" * 65) + + +def test_vue_label_strips_markup() -> None: + """Vue 文本节点用：尖括号/花括号被剔除并完成字符串级转义。""" + label = vue_label("{{ evil }}") + assert "<" not in label + assert ">" not in label + assert "{" not in label + assert "script" in label # 文本保留，只是失去了标记语义 + + +def test_assert_python_syntax_raises_for_broken_output() -> None: + """语法网：破损产物报错并带上文件名与行号。""" + assert_python_syntax("x = 1\n") # 合法不抛 + with pytest.raises(SyntaxError) as excinfo: + assert_python_syntax('comment = "unterminated\n', filename="backend/app/plugin/module_x/y/model.py") + assert "model.py" in str(excinfo.value) + + +# ── 2. 口径层：列名白名单校验 ──────────────────────────────────────────────── + + +def test_column_name_whitelist_accepts_legal_and_rejects_injection() -> None: + """列名必须能直接充当标识符：注入型/数字开头/含空格一律拒绝。""" + assert GenTableColumnOutSchema(**{**_column().model_dump(), "column_name": "order_no"}).column_name == "order_no" + assert GenTableColumnOutSchema(**{**_column().model_dump(), "column_name": ""}).column_name == "" + for bad in (EVIL_COLUMN_NAME, "2fa", "user name", "a\nb", "a'b"): + with pytest.raises(ValidationError): + GenTableColumnOutSchema(**{**_column().model_dump(), "column_name": bad}) + + +# ── 3. 渲染层：恶意输入不能产出可执行代码，且产物必须语法合法 ───────────────── + + +async def test_malicious_metadata_cannot_inject_executable_python() -> None: + """列注释/功能名/列名全为注入载荷时：产物仍可 ast.parse，且不含可执行注入节点。""" + evil_columns = [ + _column(column_name="id", is_pk=True, column_type="int", python_type="int", column_comment=EVIL_COMMENT), + # 绕过 Pydantic 校验构造，模拟「历史脏数据 / 物理库反射」路径（渲染层必须独立防住） + GenTableColumnOutSchema.model_construct(**{**_column().model_dump(), "column_name": EVIL_COLUMN_NAME}), + ] + table = _table(columns=evil_columns, function_name=EVIL_FUNCTION_NAME, table_comment=EVIL_COMMENT) + + rendered = _python_outputs(await _render_all(table)) + assert rendered, "应至少渲染出一个 Python 产物" + + for name, content in rendered.items(): + # ① 语法合法 + ast.parse(content) + # ② 无 import os / os.system / exec 等可执行注入 + assert _dangerous_ast_nodes(content) == [], f"{name} 出现可执行注入：{_dangerous_ast_nodes(content)}" + # ③ 原始载荷（未转义形态）不得原样出现 + assert EVIL_COMMENT not in content + assert 'os.system("id")' not in content + + +async def test_malicious_inputs_keep_output_syntactically_valid_for_all_python_templates() -> None: + """全部 .py 模板逐个校验语法（含 model/schema/controller/service/crud/__init__）。""" + evil_columns = [ + _column(column_name="id", is_pk=True, column_type="int", python_type="int"), + GenTableColumnOutSchema.model_construct( + **{**_column().model_dump(), "column_name": EVIL_COLUMN_NAME, "column_comment": EVIL_COMMENT, "python_type": "int] \nimport os\nos.system('id')"}, + ), + ] + table = _table(columns=evil_columns, function_name=EVIL_FUNCTION_NAME, class_name="Demo", table_name='gen_demo"); import os; #') + + rendered = _python_outputs(await _render_all(table)) + for name, content in rendered.items(): + assert_python_syntax(content, filename=name) + assert _dangerous_ast_nodes(content) == [], f"{name} 出现可执行注入" + + +async def test_escaped_comment_preserves_readable_text() -> None: + """安全化不能丢内容：普通文本（含撇号）在产物里按原义保留。""" + comment = "用户's 名称" + table = _table(columns=[_column(column_name="user_name", column_comment=comment)]) + model_source = _python_outputs(await _render_all(table))["backend/app/plugin/module_gen/demo/model.py"] + + tree = ast.parse(model_source) + comments: list[Any] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Call): + for keyword in node.keywords: + if keyword.arg == "comment": + comments.append(ast.literal_eval(keyword.value)) + assert comment in comments, f"注释文本未被原义保留：{comments}" + + +async def test_vue_template_escapes_label_positions() -> None: + """Vue 文本节点：子表功能名里的标签/脚本不能改变生成页面的标记结构。""" + sub = _table(columns=[_column(column_name="sub_name")], function_name="") + parent = _table( + columns=[_column(column_name="id", is_pk=True, column_type="int", python_type="int")], + sub=True, + sub_table=sub, + sub_table_name="gen_sub", + sub_table_fk_name="demo_id", + ) + rendered = await _render_all(parent) + vue_source = rendered["frontend/web/src/views/module_gen/demo/index.vue"] + assert "\n" + with pytest.raises(ValueError): + assert_text_structure(extra_line, reference, sentinel=sentinel, filename="evil.vue") + + breakout = 'const a = "x"; alert(1); //";\n__S__列表\n' + with pytest.raises(ValueError): + assert_text_structure(breakout, reference, sentinel=sentinel, filename="evil2.vue") + + +async def test_all_templates_pass_structural_gate_with_hostile_text() -> None: + """端到端：恶意文本下，全部模板（主+子）都能通过与落盘路径相同的结构闸门。""" + sub_cols = [ + _column(column_name="id", is_pk=True, column_type="int", python_type="int", column_comment="子主键"), + _column(column_name="sub_name", column_comment="子表'备注' & "), + ] + sub = _table( + columns=sub_cols, + table_name="gen_sub", + class_name="Sub", + business_name="sub", + function_name="", + ) + main_cols = [_column(column_name="id", is_pk=True, column_type="int", python_type="int")] + parent = _table( + columns=main_cols, + function_name=EVIL_FUNCTION_NAME, + table_comment=EVIL_COMMENT, + sub=True, + sub_table=sub, + sub_table_name="gen_sub", + sub_table_fk_name="demo_id", + ) + env = Jinja2TemplateUtil.get_env() + safe = Jinja2TemplateUtil.safe_render_schema(parent) + ctx = Jinja2TemplateUtil.prepare_context(parent) + reference_ctx = Jinja2TemplateUtil.prepare_context(Jinja2TemplateUtil.sentinel_render_schema(safe)) + sub_ctx = Jinja2TemplateUtil.prepare_sub_render_context(safe, safe.sub_table) + sub_reference_ctx = Jinja2TemplateUtil.prepare_sub_render_context( + Jinja2TemplateUtil.sentinel_render_schema(safe), + Jinja2TemplateUtil.sentinel_render_schema(safe.sub_table), + ) + tokens = Jinja2TemplateUtil.iter_render_dynamic_values(safe) + + checked = 0 + for template in Jinja2TemplateUtil.get_template_list(): + name = Jinja2TemplateUtil.get_file_name(template, parent) + real = await env.get_template(template).render_async(**ctx) + ref = await env.get_template(template).render_async(**reference_ctx) + if name.endswith(".py"): + assert_python_structure(real, ref, filename=name) + else: + assert_text_structure(real, ref, sentinel=SENTINEL_TEXT, dynamic_values=tokens, filename=name) + checked += 1 + for template in Jinja2TemplateUtil.get_sub_table_template_list(): + name = Jinja2TemplateUtil.get_file_name(template, safe.sub_table) + real = await env.get_template(template).render_async(**sub_ctx) + ref = await env.get_template(template).render_async(**sub_reference_ctx) + assert_python_structure(real, ref, filename=name) + checked += 1 + assert checked == 13 # 10 主模板 + 3 子表模板 diff --git a/backend/tests/test_scheduler_schedulability.py b/backend/tests/test_scheduler_schedulability.py new file mode 100644 index 00000000..c2ec89ac --- /dev/null +++ b/backend/tests/test_scheduler_schedulability.py @@ -0,0 +1,328 @@ +"""定时任务可调度性回归测试（t15）。 + +覆盖四类情形： +1) ``SCHEDULER_ALLOW_CODE_EXEC=false`` 时，``builtin:`` 引用型任务**可以**创建/注册（不经 exec）； +2) 同一开关下，原始代码块型任务仍被拒绝，且提示里给出 ``builtin:`` 替代写法； +3) 节点/调度器任务列表返回 ``schedulable`` 状态字段，能区分「已启用但不可调度」与「正常可调度」； +4) 运行期把「已启用但不可调度」暴露为 WARN 日志（带节点/任务标识），不再静默失效。 +""" + +from types import SimpleNamespace +from typing import Any + +import pytest +from apscheduler.triggers.date import DateTrigger +from loguru import logger + +from app.common.enums import RET # noqa: F401 (保持与其它测试一致的导入面) +from app.config.setting import settings +from app.core.ap_scheduler import SchedulerUtil +from app.core.base_schema import AuthSchema +from app.core.exceptions import CustomException +from app.modules.task.cronjob.job.schema import SchedulerJobSchema, SchedulerStatusSchema +from app.modules.task.cronjob.job.service import JobService +from app.modules.task.cronjob.node import service as node_service +from app.modules.task.cronjob.node.schema import NodeOutSchema +from app.modules.task.cronjob.node.service import _add_job_with_trigger + +# ── 工具 ────────────────────────────────────────────────────────────────────── + + +class _EmptyScalars: + """空结果集（节点列举时的执行日志查询用）。""" + + def scalars(self) -> "_EmptyScalars": + return self + + def first(self) -> None: + return None + + def all(self) -> list: + return [] + + +class _EmptySession: + """execute 恒返回空结果的假会话。""" + + async def execute(self, *_args: object, **_kwargs: object) -> _EmptyScalars: + return _EmptyScalars() + + +class _CapturedLogs: + """临时 loguru sink：收集 WARN 及以上日志文本。""" + + def __init__(self) -> None: + self.records: list[str] = [] + + def __call__(self, message: Any) -> None: + self.records.append(message.record["message"] if hasattr(message, "record") else str(message)) + + def text(self) -> str: + return "\n".join(self.records) + + +def _node(**overrides: Any) -> SimpleNamespace: + """构造一个可用于注册的节点（不落库）。""" + data: dict[str, Any] = { + "id": 101, + "name": "单元测试节点", + "func": "builtin:demo_handler", + "jobstore": "memory", + "executor": "threadpool", + "args": None, + "kwargs": None, + "coalesce": False, + "max_instances": 1, + } + data.update(overrides) + return SimpleNamespace(**data) + + +def _cleanup_jobs(*job_ids: str) -> None: + """清理注册到调度器里的测试任务（幂等）。""" + for job_id in job_ids: + try: + SchedulerUtil.remove_job(job_id=job_id) + except Exception: + pass + + +# ── 1) builtin 引用：开关关闭时仍可创建/注册 ────────────────────────────────── + + +def test_builtin_reference_registers_while_code_exec_disabled(monkeypatch: pytest.MonkeyPatch) -> None: + """开关关闭时，builtin: 引用型任务能成功注册（且执行路径不经 exec）。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + _cleanup_jobs("unit-builtin-reg") + try: + job = _add_job_with_trigger(_node(id="unit-builtin-reg"), DateTrigger(run_date=None)) + assert str(job.id) == "unit-builtin-reg" + # 执行入口同样放行，且返回内置处理器的结果（证明没走 exec） + result = SchedulerUtil._task_wrapper("unit-builtin-reg", job.args[1]) + assert isinstance(result, dict) and result.get("message") + finally: + _cleanup_jobs("unit-builtin-reg") + + +def test_builtin_reference_is_reported_as_schedulable(monkeypatch: pytest.MonkeyPatch) -> None: + """可调度性判定：builtin: 引用为可调度，未知内置则不可调度并给出原因。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + assert SchedulerUtil.describe_schedulability("builtin:demo_handler") == (True, None) + schedulable, reason = SchedulerUtil.describe_schedulability("builtin:not_exist.handler") + assert schedulable is False + assert reason and "内置处理器不可用" in reason + + +# ── 2) 原始代码块：开关关闭时仍被拒绝，并提示 builtin 替代 ───────────────────── + + +def test_raw_code_block_is_rejected_with_builtin_hint(monkeypatch: pytest.MonkeyPatch) -> None: + """原始代码块被拒绝，提示必须点명 SCHEDULER_ALLOW_CODE_EXEC 与 builtin: 替代写法。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + node = _node(func="def handler(*args, **kwargs):\n return 1\n") + with pytest.raises(CustomException) as excinfo: + _add_job_with_trigger(node, DateTrigger(run_date=None)) + message = str(excinfo.value) + assert "SCHEDULER_ALLOW_CODE_EXEC" in message + assert "builtin:" in message + assert excinfo.value.status_code == 400 # 客户端/策略类拒绝，不是 500 + + +def test_raw_code_block_still_allowed_when_explicitly_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + """开关显式打开时，原始代码块仍可注册（本次改动没有收紧既有能力）。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", True) + _cleanup_jobs("unit-raw-reg") + try: + job = _add_job_with_trigger(_node(id="unit-raw-reg", func="def handler(*a, **k):\n return 'ok'\n"), DateTrigger(run_date=None)) + assert str(job.id) == "unit-raw-reg" + assert SchedulerUtil.describe_schedulability(SchedulerUtil.job_code_block(job)) == (True, None) + finally: + _cleanup_jobs("unit-raw-reg") + + +def test_empty_func_is_rejected_before_schedulability_check(monkeypatch: pytest.MonkeyPatch) -> None: + """空 func 仍是"不能为空"（保持既有报错语义），不被可调度性信息覆盖。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + with pytest.raises(ValueError, match="不能为空"): + _add_job_with_trigger(_node(func=" "), DateTrigger(run_date=None)) + + +# ── 3) 列表状态字段：区分「已启用但不可调度」与「正常可调度」 ───────────────────── + + +async def test_node_list_exposes_schedulable_status(monkeypatch: pytest.MonkeyPatch) -> None: + """节点列表：原始代码块 + 开关关闭 → schedulable=false 且带原因；builtin → true。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + node_service._WARNED_UNSCHEDULABLE.clear() + monkeypatch.setattr(SchedulerUtil, "get_jobs", classmethod(lambda cls, jobstore=None: [])) + + service = node_service.NodeService(AuthSchema(), _EmptySession()) # type: ignore[arg-type] + raw_item = NodeOutSchema(id=201, name="原始代码块节点", status=0, func="def handler(): pass") + builtin_item = NodeOutSchema(id=202, name="内置函数节点", status=0, func="builtin:demo_handler") + await service._enrich_runtime([raw_item, builtin_item]) + + assert raw_item.schedulable is False + assert raw_item.unschedulable_reason and "builtin:" in raw_item.unschedulable_reason + assert builtin_item.schedulable is True + assert builtin_item.unschedulable_reason is None + + +async def test_node_list_marks_disabled_node_schedulable_but_via_status(monkeypatch: pytest.MonkeyPatch) -> None: + """停用节点（status=1）不触发告警，但仍如实返回 schedulable 字段。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + node_service._WARNED_UNSCHEDULABLE.clear() + monkeypatch.setattr(SchedulerUtil, "get_jobs", classmethod(lambda cls, jobstore=None: [])) + logs = _CapturedLogs() + sink_id = logger.add(logs, level="WARNING") + + service = node_service.NodeService(AuthSchema(), _EmptySession()) # type: ignore[arg-type] + disabled_item = NodeOutSchema(id=203, name="停用节点", status=1, func="def handler(): pass") + try: + await service._enrich_runtime([disabled_item]) + finally: + logger.remove(sink_id) + + assert disabled_item.schedulable is False + assert "id=203" not in logs.text() # 未启用不告警，避免噪音 + + +def test_scheduler_job_list_and_status_expose_schedulability(monkeypatch: pytest.MonkeyPatch) -> None: + """调度器任务列表 / 状态：带 schedulable 与 unschedulable_count（已注册但不可执行）。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", True) + _cleanup_jobs("unit-sched-raw") + try: + _add_job_with_trigger( + _node(id="unit-sched-raw", func="def handler(*a, **k):\n return 'ok'\n"), + DateTrigger(run_date=None), + ) + # 打开时：可调度 + items = {item.id: item for item in JobService.get_scheduler_jobs()} + assert items["unit-sched-raw"].schedulable is True + assert JobService.get_scheduler_status().unschedulable_count == 0 + + # 关闭后：同一任务变成不可执行，且状态里计数可见 + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + items = {item.id: item for item in JobService.get_scheduler_jobs()} + assert items["unit-sched-raw"].schedulable is False + assert items["unit-sched-raw"].unschedulable_reason + status = JobService.get_scheduler_status() + assert status.code_exec_enabled is False + assert status.unschedulable_count >= 1 + finally: + _cleanup_jobs("unit-sched-raw") + + +def test_scheduler_schemas_have_expected_fields() -> None: + """契约字段存在性（前端/运维依赖）。""" + assert {"status", "is_running", "job_count", "code_exec_enabled", "unschedulable_count"} <= set(SchedulerStatusSchema.model_fields) + assert {"id", "name", "trigger", "next_run_time", "status", "schedulable", "unschedulable_reason"} <= set(SchedulerJobSchema.model_fields) + assert {"schedulable", "unschedulable_reason"} <= set(NodeOutSchema.model_fields) + + +# ── 4) 运行期可见性：WARN 日志带具体标识，不静默失效 ───────────────────────── + + +async def test_unschedulable_enabled_node_logs_warning_with_identifier(monkeypatch: pytest.MonkeyPatch) -> None: + """已启用但不可调度的节点：WARN 日志必须带节点 id 与原因，且同一原因只打一次。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + node_service._WARNED_UNSCHEDULABLE.clear() + monkeypatch.setattr(SchedulerUtil, "get_jobs", classmethod(lambda cls, jobstore=None: [])) + logs = _CapturedLogs() + sink_id = logger.add(logs, level="WARNING") + + service = node_service.NodeService(AuthSchema(), _EmptySession()) # type: ignore[arg-type] + item = NodeOutSchema(id=301, name="不可调度节点", status=0, func="def handler(): pass") + try: + await service._enrich_runtime([item]) + await service._enrich_runtime([item]) # 第二次不应重复告警 + finally: + logger.remove(sink_id) + + text = logs.text() + assert "id=301" in text + assert "不可调度" in text + assert "SCHEDULER_ALLOW_CODE_EXEC" in text + assert text.count("id=301") == 1, f"同一节点同一原因重复告警：{text}" + + +def test_scheduler_status_logs_warning_for_registered_but_dead_jobs(monkeypatch: pytest.MonkeyPatch) -> None: + """调度器状态查询：已注册但不可执行的任务要打 WARN（带 job id）。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", True) + _cleanup_jobs("unit-sched-log") + try: + _add_job_with_trigger( + _node(id="unit-sched-log", func="def handler(*a, **k):\n return 'ok'\n"), + DateTrigger(run_date=None), + ) + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + logs = _CapturedLogs() + sink_id = logger.add(logs, level="WARNING") + try: + status = JobService.get_scheduler_status() + finally: + logger.remove(sink_id) + assert status.unschedulable_count >= 1 + assert "unit-sched-log" in logs.text() + finally: + _cleanup_jobs("unit-sched-log") + + +# ── 5) 创建/更新/手动执行入口（create/update → register_node_job）────────────── + + +def test_register_node_job_accepts_builtin_when_disabled(monkeypatch: pytest.MonkeyPatch) -> None: + """创建/更新路径（register_node_job）在开关关闭时能注册 builtin: 引用型任务。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + node = _node( + id="unit-reg-builtin", + trigger="date", + trigger_args="2099-01-01 00:00:00", + start_date=None, + end_date=None, + status=0, + ) + try: + job = node_service.register_node_job(node) + assert job is not None and str(job.id) == "unit-reg-builtin" + finally: + node_service.unregister_node_job("unit-reg-builtin") + + +def test_register_node_job_rejects_raw_code_when_disabled(monkeypatch: pytest.MonkeyPatch) -> None: + """创建/更新路径在开关关闭时拒绝原始代码块，并给出 builtin 替代写法（不再静默不注册）。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + node = _node( + id="unit-reg-raw", + trigger="date", + trigger_args="2099-01-01 00:00:00", + start_date=None, + end_date=None, + status=0, + func="def handler(*args, **kwargs):\n return 1\n", + ) + with pytest.raises(CustomException) as excinfo: + node_service.register_node_job(node) + message = str(excinfo.value) + assert "SCHEDULER_ALLOW_CODE_EXEC" in message and "builtin:" in message + assert SchedulerUtil.get_job(job_id="unit-reg-raw") is None # 未注册成功 + + +def test_run_node_once_allows_builtin_when_disabled(monkeypatch: pytest.MonkeyPatch) -> None: + """手动执行入口：开关关闭时 builtin 引用仍可排程执行。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + node = _node(id="unit-once-builtin", trigger=None, trigger_args=None, start_date=None, end_date=None, status=0) + temp_job_id = node_service.run_node_once(node) + try: + assert temp_job_id.startswith("unit-once-builtin") + assert SchedulerUtil.get_job(job_id=temp_job_id) is not None + finally: + node_service.unregister_node_job(temp_job_id) + + +def test_run_node_once_rejects_raw_code_when_disabled(monkeypatch: pytest.MonkeyPatch) -> None: + """手动执行入口：开关关闭时原始代码块被拒绝并提示 builtin 写法。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + node = _node(id="unit-once-raw", trigger=None, func="def handler(): return 1", status=0) + with pytest.raises(CustomException) as excinfo: + node_service.run_node_once(node) + assert "builtin:" in str(excinfo.value) diff --git a/backend/tests/test_security_defaults.py b/backend/tests/test_security_defaults.py new file mode 100644 index 00000000..05ed37be --- /dev/null +++ b/backend/tests/test_security_defaults.py @@ -0,0 +1,516 @@ +"""安全默认值与异常状态码回归测试。 + +覆盖 backend/audit-security.md 的 S9/S10 与 backend/audit-backend.md 的 H1/H5 同类根因： +- 业务异常按自身状态码返回：客户端可修正类（参数/验证码/权限/不存在）为 4xx； +- 「包装底层异常」的内部故障（DB/驱动/Redis/SDK）显式为 500，不得降级成 4xx（否则监控误判）； +- 定时任务代码块执行默认关闭，关闭时给出明确失败原因（而非静默执行/静默跳过）； +- 生产环境 CORS 不再回落为通配 ``["*"]``（与 ``ALLOW_CREDENTIALS=True`` 组合会放行任意站点带凭据跨域）。 +""" + +import pathlib +import re +from datetime import datetime +from types import SimpleNamespace + +import pytest +from apscheduler.triggers.date import DateTrigger +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from app.common.enums import RET, EnvironmentEnum +from app.config.setting import Settings, settings +from app.core.ap_scheduler import SchedulerUtil +from app.core.exceptions import CustomException, handle_exception, resolve_http_status +from app.core.middlewares import CustomCORSMiddleware +from app.modules.task.cronjob.node.service import _add_job_with_trigger + +# ── 1. 业务异常按自身状态码返回（默认 4xx） ────────────────────────────────── + + +def test_custom_exception_defaults_to_4xx() -> None: + """未显式指定状态码时按业务错误处理，而不是 500。""" + exc = CustomException(msg="验证码已过期，请刷新") + assert exc.status_code == 400 + assert exc.code == RET.EXCEPTION.code # 业务码保持不变（响应体结构兼容） + assert exc.success is False + + +def test_custom_exception_status_resolution() -> None: + """显式状态码优先；否则按业务码映射；未知码回落 400。""" + assert CustomException(msg="x", status_code=403).status_code == 403 + assert CustomException(msg="x", status_code=500).status_code == 500 + assert CustomException(msg="x", code=RET.UNAUTHORIZED.code).status_code == 401 + assert CustomException(msg="x", code=RET.NOT_FOUND.code).status_code == 404 + assert CustomException(msg="x", code=RET.NO_PERMISSION.code).status_code == 403 + assert CustomException(msg="x", code=RET.TOO_MANY_REQUESTS.code).status_code == 429 + assert CustomException(msg="x", code=RET.SERVERERR.code).status_code == 500 + assert resolve_http_status(None) == 400 + assert resolve_http_status(999999) == 400 + + +def _exception_probe_app() -> FastAPI: + """最小应用：注册与生产一致的异常处理器。""" + app = FastAPI() + handle_exception(app) + + @app.get("/captcha-expired") + async def _captcha_expired() -> None: + raise CustomException(msg="验证码已过期，请刷新") + + @app.get("/forbidden") + async def _forbidden() -> None: + raise CustomException(msg="无权限操作", code=RET.NO_PERMISSION.code, status_code=403) + + @app.get("/boom") + async def _boom() -> None: + raise RuntimeError("unexpected server failure") + + return app + + +def test_captcha_expired_returns_4xx_with_stable_body() -> None: + """实测回归：验证码过期返回 400（此前为 500），响应体结构保持既有约定。""" + client = TestClient(_exception_probe_app(), raise_server_exceptions=False) + response = client.get("/captcha-expired") + assert response.status_code == 400, response.text + body = response.json() + assert set(body) >= {"code", "msg", "data", "status_code", "success"} + assert body["msg"] == "验证码已过期，请刷新" + assert body["success"] is False + assert body["status_code"] == 400 + + +def test_explicit_status_code_is_honored_by_handler() -> None: + """显式状态码仍按原样返回（403 不被 400 覆盖）。""" + client = TestClient(_exception_probe_app(), raise_server_exceptions=False) + response = client.get("/forbidden") + assert response.status_code == 403, response.text + assert response.json()["code"] == RET.NO_PERMISSION.code + + +def test_unhandled_exception_still_returns_500() -> None: + """未捕获的服务端异常仍然是 500（本次修改没有把真正的故障降级成 4xx）。""" + client = TestClient(_exception_probe_app(), raise_server_exceptions=False) + response = client.get("/boom") + assert response.status_code == 500, response.text + + +# ── 2. 定时任务代码块执行默认关闭 ──────────────────────────────────────────── + + +def test_scheduler_code_exec_default_is_false() -> None: + """配置默认值必须是 False（只在显式配置为 true 时启用）。""" + assert Settings.model_fields["SCHEDULER_ALLOW_CODE_EXEC"].default is False + + +def test_task_wrapper_refuses_code_block_with_clear_message(monkeypatch: pytest.MonkeyPatch) -> None: + """关闭时代码块任务在**执行入口**失败，并给出可理解的拒绝原因。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + with pytest.raises(RuntimeError) as excinfo: + SchedulerUtil._task_wrapper("job-1", "def handler(*args, **kwargs):\n return 'executed'\n") + message = str(excinfo.value) + assert "SCHEDULER_ALLOW_CODE_EXEC" in message + assert "拒绝执行" in message + + +def test_task_wrapper_runs_only_when_explicitly_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + """显式开启后代码块才被执行（证明开关语义正确、不是"永远拒绝"）。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", True) + result = SchedulerUtil._task_wrapper("job-2", "def handler(*args, **kwargs):\n return 'executed'\n") + assert result == "executed" + + +def test_task_wrapper_without_code_block_still_allowed(monkeypatch: pytest.MonkeyPatch) -> None: + """内置函数型任务（无用户代码块）不受该开关影响。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + assert SchedulerUtil._task_wrapper("job-3", None) is None + + +def test_registering_code_block_job_reports_clear_error_when_disabled(monkeypatch: pytest.MonkeyPatch) -> None: + """注册入口同样给出明确失败提示（而不是把任务静默注册后不执行）。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + node = SimpleNamespace( + id=7, + name="演示节点", + func="def handler(*args, **kwargs):\n return 'x'\n", + jobstore="memory", + executor="threadpool", + args=None, + kwargs=None, + coalesce=False, + max_instances=1, + ) + with pytest.raises(CustomException) as excinfo: + _add_job_with_trigger(node, DateTrigger(run_date=datetime.now())) + assert "SCHEDULER_ALLOW_CODE_EXEC" in str(excinfo.value) + + +# ── 3. 生产 CORS 不再回落通配 ──────────────────────────────────────────────── + + +def _cors_probe_app() -> FastAPI: + app = FastAPI() + app.add_middleware(CustomCORSMiddleware) + + @app.get("/probe") + async def _probe() -> dict[str, bool]: + return {"ok": True} + + return app + + +def test_prod_cors_is_empty_when_not_configured(monkeypatch: pytest.MonkeyPatch) -> None: + """生产未配置 PROD_CORS_ORIGINS → 空列表（安全默认），不再是 ["*"]。""" + monkeypatch.setattr(settings, "ENVIRONMENT", EnvironmentEnum.PROD) + monkeypatch.setattr(settings, "PROD_CORS_ORIGINS", "") + assert settings.ALLOW_ORIGINS == [] + + +def test_prod_cors_drops_wildcard(monkeypatch: pytest.MonkeyPatch) -> None: + """显式配置通配符也要被剔除（与 ALLOW_CREDENTIALS=True 组合不可接受）。""" + monkeypatch.setattr(settings, "ENVIRONMENT", EnvironmentEnum.PROD) + monkeypatch.setattr(settings, "PROD_CORS_ORIGINS", "*") + assert settings.ALLOW_ORIGINS == [] + monkeypatch.setattr(settings, "PROD_CORS_ORIGINS", "https://ok.example.com, *") + assert settings.ALLOW_ORIGINS == ["https://ok.example.com"] + + +def test_prod_cors_keeps_explicit_origins(monkeypatch: pytest.MonkeyPatch) -> None: + """显式配置的域名清单被保留（去掉空白项）。""" + monkeypatch.setattr(settings, "ENVIRONMENT", EnvironmentEnum.PROD) + monkeypatch.setattr(settings, "PROD_CORS_ORIGINS", " https://a.example.com , https://b.example.com , ") + assert settings.ALLOW_ORIGINS == ["https://a.example.com", "https://b.example.com"] + + +def test_dev_cors_keeps_wildcard(monkeypatch: pytest.MonkeyPatch) -> None: + """非生产环境保留通配（本地联调便利），安全默认只收紧生产。""" + monkeypatch.setattr(settings, "ENVIRONMENT", EnvironmentEnum.DEV) + assert settings.ALLOW_ORIGINS == ["*"] + + +def test_prod_cors_headers_absent_for_unknown_origin(monkeypatch: pytest.MonkeyPatch) -> None: + """端到端：生产未配置域名时，任意来源都拿不到跨域许可头。""" + monkeypatch.setattr(settings, "ENVIRONMENT", EnvironmentEnum.PROD) + monkeypatch.setattr(settings, "PROD_CORS_ORIGINS", "") + client = TestClient(_cors_probe_app()) + response = client.get("/probe", headers={"Origin": "https://evil.tld"}) + assert response.status_code == 200 + assert "access-control-allow-origin" not in {key.lower() for key in response.headers} + + +def test_prod_cors_allows_only_configured_origin(monkeypatch: pytest.MonkeyPatch) -> None: + """端到端：只有显式配置的来源拿到许可头，其它来源仍被拒绝。""" + monkeypatch.setattr(settings, "ENVIRONMENT", EnvironmentEnum.PROD) + monkeypatch.setattr(settings, "PROD_CORS_ORIGINS", "https://ok.example.com") + client = TestClient(_cors_probe_app()) + + allowed = client.get("/probe", headers={"Origin": "https://ok.example.com"}) + assert allowed.headers.get("access-control-allow-origin") == "https://ok.example.com" + + denied = client.get("/probe", headers={"Origin": "https://evil.tld"}) + assert "access-control-allow-origin" not in {key.lower() for key in denied.headers} + + +# ── 4. 内部故障（包装底层异常）必须为 500，客户端错误保持 4xx ───────────────── +# +# 背景：t10 把 CustomException 的默认状态码从 500 改为 400 后，「包装底层异常」的路径 +# （数据库/驱动/Redis/SDK 故障）也被一并降级成 400，监控会把服务端故障误判为客户端错误。 +# 这里固化两类语义：包装底层异常 → 500；参数/权限/不存在 → 4xx。 + +_REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent + + +class _FailingSession: + """所有 SQL 操作都抛异常的假会话：模拟数据库/驱动故障。""" + + @staticmethod + def _boom(*_args: object, **_kwargs: object) -> None: + raise RuntimeError("模拟底层驱动故障: connection lost") + + def add(self, *_args: object, **_kwargs: object) -> None: + self._boom() + + async def execute(self, *_args: object, **_kwargs: object) -> None: + self._boom() + + async def flush(self, *_args: object, **_kwargs: object) -> None: + self._boom() + + async def refresh(self, *_args: object, **_kwargs: object) -> None: + self._boom() + + async def delete(self, *_args: object, **_kwargs: object) -> None: + self._boom() + + +class _EmptyResult: + """空结果集：让 get_or_404 走到"查不到"分支。""" + + def scalars(self) -> "_EmptyResult": + return self + + def first(self) -> None: + return None + + def all(self) -> list: + return [] + + def scalar(self) -> int: + return 0 + + def fetchall(self) -> list: + return [] + + +class _EmptySession: + """execute 恒返回空结果的假会话。""" + + async def execute(self, *_args: object, **_kwargs: object) -> _EmptyResult: + return _EmptyResult() + + +def _crud(db: object, model: type | None = None): + """构造 CRUD 实例（默认挂在一个真实模型上，避免测试私自注册新模型污染 metadata）。""" + from app.core.base_crud import CRUDBase + from app.core.base_schema import AuthSchema + from app.modules.system.role.model import RoleModel + + return CRUDBase(model=model or RoleModel, auth=AuthSchema(), db=db) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("case", "call"), + [ + ("get", lambda c: c.get(id=1)), + ("count", lambda c: c.count()), + ("exists", lambda c: c.exists(id=1)), + ("get_list", lambda c: c.get_list()), + ("page", lambda c: c.page(offset=0, limit=10, order_by=[{"id": "asc"}])), + ("create", lambda c: c.create(data={"name": "x"})), + ("update", lambda c: c.update(id=1, data={"name": "x"})), + ("delete", lambda c: c.delete(ids=[1])), + ("clear", lambda c: c.clear()), + ("set", lambda c: c.set(ids=[1], status=0)), + ], +) +async def test_base_crud_wrapped_internal_failure_returns_500(case: str, call) -> None: + """CRUD 公共出口包装底层异常时必须 500（此前被 t10 降级成 400）。""" + with pytest.raises(CustomException) as excinfo: + await call(_crud(_FailingSession())) + assert excinfo.value.status_code == 500, f"{case} 未按 500 返回（实际 {excinfo.value.status_code}）" + assert excinfo.value.success is False + assert "失败" in excinfo.value.msg + + +async def test_get_or_404_returns_404_for_missing_row() -> None: + """资源不存在 → 404（客户端可修正类错误，仍是 4xx）。""" + with pytest.raises(CustomException) as excinfo: + await _crud(_EmptySession()).get_or_404(id=1, msg="该数据不存在") + assert excinfo.value.status_code == 404 + assert excinfo.value.msg == "该数据不存在" + + +def test_client_side_errors_keep_4xx() -> None: + """客户端类错误保持 4xx：验证码过期/参数类 400、无权限 403、资源不存在 404。""" + assert CustomException(msg="验证码已过期，请刷新").status_code == 400 + assert CustomException(msg="请求参数错误").status_code == 400 + assert CustomException(msg="无权限操作", code=RET.NO_PERMISSION.code).status_code == 403 + assert CustomException(msg="该数据不存在", status_code=404).status_code == 404 + assert resolve_http_status(RET.NOT_FOUND.code) == 404 + assert resolve_http_status(RET.NO_PERMISSION.code) == 403 + assert resolve_http_status(RET.BAD_REQUEST.code) == 400 + assert resolve_http_status(RET.SERVERERR.code) == 500 + + +def test_internal_failure_body_keeps_business_code_and_message() -> None: + """内部故障 500 时响应体仍带原始 msg 与业务 code（前端依赖 code=-1 展示真实原因）。""" + app = FastAPI() + handle_exception(app) + + @app.get("/internal") + async def _internal() -> None: + raise CustomException(msg="统计失败: 模拟底层驱动故障", status_code=500) + + response = TestClient(app, raise_server_exceptions=False).get("/internal") + assert response.status_code == 500 + body = response.json() + assert body["msg"] == "统计失败: 模拟底层驱动故障" + assert body["code"] == RET.EXCEPTION.code + + +# 与实现同源的分类规则（见 app/core/exceptions.py 的说明与 t11 的清理范围） +_EXC_INTERP = re.compile(r"\{(?:e|exc|err|error|ex|exception)(?:!s|!r|:[a-z]+)?\}|\{errors\[|_error_desc\(") +_EXC_CALL = re.compile(r"CustomException\(") +_CLIENT_MARKERS = ( + "targets 参数格式错误", + "创建成功但任务注册失败", + "更新成功但任务注册失败", + "启用任务失败", + 'msg=f"执行失败: {e!s}"', + "本地目录不存在", + "不支持的存储协议", + "存储源不存在", + "存储源已停用", + "不允许上传此类型的文件", + "流程保存失败", + "执行失败，连线", + "创建失败，该字典类型下的", + "更新失败，该字典类型下的", + "删除失败，ID为", + "角色不存在", + "菜单不存在", + "部门不存在", + "用户不存在", + "岗位不存在", + "导入文件缺少必要的列", + "不允许从", + "工单[", +) + + +def _iter_custom_exception_calls(path: pathlib.Path): + """遍历文件里每个 CustomException(...) 调用（含多行调用）。""" + text = path.read_text(encoding="utf-8") + for match in _EXC_CALL.finditer(text): + depth, index = 0, match.end() - 1 + while index < len(text): + if text[index] == "(": + depth += 1 + elif text[index] == ")": + depth -= 1 + if depth == 0: + break + index += 1 + yield text[: match.start()].count("\n") + 1, text[match.start() : index + 1] + + +def test_all_wrapped_internal_failures_are_explicitly_500() -> None: + """守卫：inScope 内凡是「包装底层异常」的 CustomException 都必须显式 500。 + + 否则底层故障会被默认值（400）静默降级成客户端错误，监控失去信号。 + """ + offenders: list[str] = [] + checked = 0 + targets = [_REPO_ROOT / "app" / "core" / "base_crud.py"] + sorted((_REPO_ROOT / "app" / "modules").rglob("*.py")) + for path in targets: + for line, call in _iter_custom_exception_calls(path): + if not _EXC_INTERP.search(call): + continue + if any(marker in call for marker in _CLIENT_MARKERS): + continue + checked += 1 + if "status_code=500" not in call: + offenders.append(f"{path.relative_to(_REPO_ROOT)}:{line}") + assert checked > 90, f"分类用例数异常（只匹配到 {checked} 处），分类规则可能失效" + assert offenders == [], f"以下代码生成路径仍可能把内部故障降级为 4xx：{offenders}" + + +def test_base_crud_internal_wrappers_are_500() -> None: + """core 层专门固化：base_crud.py 全部 9 处包装路径显式 500。""" + calls = [ + (line, call) + for line, call in _iter_custom_exception_calls(_REPO_ROOT / "app" / "core" / "base_crud.py") + if _EXC_INTERP.search(call) + ] + assert len(calls) == 9, f"base_crud.py 包装路径数量变化（{len(calls)}）：{[(line, call[:30]) for line, call in calls]}" + missing = [line for line, call in calls if "status_code=500" not in call] + assert missing == [], f"base_crud.py 以下行未显式 500：{missing}" + + +# ── 5. 升级可见性：CORS 严格启动、主机放行开关、健康检查字段 ────────────────── +# +# 背景：t13 把三处「静默行为变更」做成可配置 + 可观测： +# - CORS_STRICT_STARTUP：生产未配置 CORS 白名单时从「只告警」变为「启动失败」； +# - ALLOW_LOCALHOST_HOSTS：localhost 放行从无条件写死变为可开关； +# - 健康检查暴露 code_exec_enabled / cors_origins_configured，避免升级后「任务不动了」无迹可循。 + + +def test_cors_strict_startup_fails_fast(monkeypatch: pytest.MonkeyPatch) -> None: + """严格模式下：生产未配置白名单 → 直接抛错（启动失败），而不是静默告警。""" + strict = Settings( + _env_file=None, + ENVIRONMENT=EnvironmentEnum.PROD, + PROD_CORS_ORIGINS="", + CORS_STRICT_STARTUP=True, + ) + with pytest.raises(RuntimeError) as excinfo: + _ = strict.ALLOW_ORIGINS + assert "PROD_CORS_ORIGINS" in str(excinfo.value) + + # 默认（非严格）仍是安全默认 + 告警，不阻断启动 + lax = Settings(_env_file=None, ENVIRONMENT=EnvironmentEnum.PROD, PROD_CORS_ORIGINS="", CORS_STRICT_STARTUP=False) + assert lax.ALLOW_ORIGINS == [] + + +def test_localhost_hosts_are_configurable() -> None: + """localhost/回环放行可开关：默认兼容容器健康检查，生产可收敛 Host 校验范围。""" + default = Settings(_env_file=None, ALLOW_LOCALHOST_HOSTS=True) + for host in ("localhost", "127.0.0.1", "[::1]"): + assert host in default.ALLOWED_HOSTS + + restricted = Settings(_env_file=None, ALLOW_LOCALHOST_HOSTS=False) + assert "localhost" not in restricted.ALLOWED_HOSTS + assert "127.0.0.1" not in restricted.ALLOWED_HOSTS + assert restricted.ALLOWED_HOSTS == ["service.fastapiadmin.com", "*.fastapiadmin.com"] + + +def test_oauth_allowed_hosts_default_is_not_wildcard() -> None: + """OAuth 回调域名白名单默认不再是 "*"（Host 注入不再自动放行）。""" + defaults = Settings(_env_file=None) + assert "*" not in defaults.OAUTH_ALLOWED_HOSTS + assert "service.fastapiadmin.com" in defaults.OAUTH_ALLOWED_HOSTS + + +def test_health_exposes_scheduler_and_cors_state() -> None: + """健康检查暴露「代码执行开关 / CORS 是否已配置」，让升级后的行为变更可观测。""" + from app.modules.monitor.health.schema import ServiceInfoOut + + fields = ServiceInfoOut.model_fields + assert "code_exec_enabled" in fields + assert "cors_origins_configured" in fields + + +async def test_health_collect_reports_code_exec_flag(monkeypatch: pytest.MonkeyPatch) -> None: + """collect() 实际填充该字段（与 settings 联动）。""" + from app.modules.monitor.health.service import HealthService + + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + info = await HealthService.collect(None) + assert info.code_exec_enabled is False + assert set(info.model_dump()) >= {"name", "version", "environment", "db_status", "redis_status", "code_exec_enabled", "cors_origins_configured"} + + +# ── 6. 内置函数型任务：开关只关闭 exec，不关闭整个调度特性 ───────────────────── +# +# 背景：SCHEDULER_ALLOW_CODE_EXEC 默认 False 后，若没有"非 exec 的合法任务路径"， +# 升级等于停掉全部节点型定时任务。内置处理器白名单（func = "builtin:<模块名>[.<函数名>]"） +# 提供不经 exec 的执行路径，因此该开关只收敛 RCE 面。 + + +def test_builtin_handler_runs_without_code_exec(monkeypatch: pytest.MonkeyPatch) -> None: + """代码执行关闭时，内置函数型任务仍可正常调度（不经 exec）。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + result = SchedulerUtil._task_wrapper("job-builtin", "builtin:demo_handler") + assert isinstance(result, dict) and result.get("message") + + +def test_builtin_handler_can_be_registered_explicitly(monkeypatch: pytest.MonkeyPatch) -> None: + """注册表可显式注册处理器（供业务侧接入自有白名单函数）。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", False) + SchedulerUtil.register_builtin_handler("unit_test_probe", lambda *a, **k: "unit-ok") + assert SchedulerUtil.resolve_builtin_handler("builtin:unit_test_probe") is not None + assert SchedulerUtil._task_wrapper("job-custom", "builtin:unit_test_probe") == "unit-ok" + + +def test_builtin_reference_never_falls_back_to_exec(monkeypatch: pytest.MonkeyPatch) -> None: + """非法/未知 builtin 引用必须报错，绝不能回退成 exec 执行。""" + monkeypatch.setattr(settings, "SCHEDULER_ALLOW_CODE_EXEC", True) # 即使开关打开也不允许回退 + for bad in ("builtin:../../etc/passwd", "builtin:nope.nope", "builtin:", "builtin:os.system"): + with pytest.raises(ValueError): + SchedulerUtil._task_wrapper("job-bad", bad) + + +def test_non_builtin_reference_is_not_treated_as_builtin() -> None: + """普通代码块不会被误判成内置引用。""" + assert SchedulerUtil.resolve_builtin_handler("def handler(): return 1") is None + assert SchedulerUtil.resolve_builtin_handler(None) is None