public/FastapiAdmin
1
0
Fork 0
mirror of https://github.com/fastapiadmin/FastapiAdmin.git synced 2026-10-15 21:35:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(backend): 安全默认值、异常语义、代码生成注入面与定时任务内置处理器

Browse Source 
本批为 t10 → t13 → t15 三轮实现 + 一轮复核驳回整改的最终形态（pytest 83 passed / ruff clean，
已部署并线上验证）。

异常语义
- CustomException 默认状态码 500 → 400（约 433 处裸调用不再被计为服务端故障），
  且「包装底层异常」的 130 处显式恢复 500；get_or_404 → 404；响应体结构不变

代码生成注入面（复核 blocker F1，二次修复）
- column_type 不再做字符串拼接：改为常量白名单派生（类型名 ∈ DB_TO_SQLALCHEMY ∪ {Boolean}，
  长度仅取数字/正则捕获），彻底移除 split("(")[1]
- 列类型加形状白名单；字符串/注释/文档字符串用 \uXXXX，标识符过 safe_ident（含 Python/JS 关键字），
  .vue 文本与静态属性改 HTML 实体
- 落盘闸门由 ast.parse 升级为 assert_python_structure（语法 + 顶层语句白名单 + 与哨兵骨架逐节点比对）
  与 assert_text_structure（.vue/.ts/.toml 行结构）；U+2028/2029 与 C1 控制符纳入转义

安全默认值
- 生产 CORS 不再回落 ["*"]（未配置即拒绝跨域，可选 CORS_STRICT_STARTUP 直接启动失败）
- OAUTH_ALLOWED_HOSTS 不再默认 "*"；新增 ALLOW_LOCALHOST_HOSTS 控制 Host 校验范围
- ALLOWED_HOSTS 收窄为按配置派生；健康检查暴露 code_exec_enabled / cors_origins_configured

定时任务
- SCHEDULER_ALLOW_CODE_EXEC 默认 False；新增 builtin:<模块名>[.<函数名>] 内置处理器路线（不经 exec）
- 判定收敛到 describe_schedulability；列表返回 schedulable / unschedulable_reason；
  启动 reconcile 与状态查询对「已启用但不可调度」打 WARN（去重、恢复后清除）
- 加固：内置处理器必须在本模块内定义或经 HANDLERS 声明，拒绝模块内导入的外部对象（复核 R1）
- 顺带修复：_WARNED_UNSCHEDULABLE 的 set.pop TypeError、job.next_run_time 缺失导致的列表 500

测试：新增 tests/test_generator_template_safety.py、test_security_defaults.py、
test_scheduler_schedulability.py（含恶意载荷、闸门红绿、静态护栏）
This commit is contained in:
zhangtao
2026-10-04 05:12:32 +08:00
parent 007864051a
commit 07fa789d2e
39 changed files with 3031 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/modules/task/cronjob/job/schema.py
+4
View File
@@ -14,6 +14,8 @@ class SchedulerStatusSchema(BaseModel):
status: str = Field(..., description="调度器状态(运行中/已暂停/已停止/未知)")
is_running: bool = Field(..., description="是否运行中")
job_count: int = Field(..., description="调度器任务数量")
code_exec_enabled: bool = Field(default=False, description="是否允许执行用户提交的代码块(SCHEDULER_ALLOW_CODE_EXEC)")
unschedulable_count: int = Field(default=0, description="已注册但当前不可执行的任务数（>0 时列表页会给出原因）")
class SchedulerJobSchema(BaseModel):
@@ -24,6 +26,8 @@ class SchedulerJobSchema(BaseModel):
trigger: str = Field(..., description="触发器")
next_run_time: str | None = Field(default=None, description="下次运行时间")
status: int = Field(..., description="任务状态(0:运行中 1:暂停中 2:已停止 3:未知)")
schedulable: bool = Field(default=True, description="当前是否可执行；false 表示已注册但执行时会被拒绝")
unschedulable_reason: str | None = Field(default=None, description="不可执行原因（含修复建议）")
class SchedulerJobModifySchema(BaseModel):
backend/app/modules/task/cronjob/job/service.py
+21 -9
View File
@@ -1,5 +1,6 @@
from sqlalchemy.ext.asyncio import AsyncSession
from app.config.setting import settings
from app.core.ap_scheduler import SchedulerUtil
from app.core.base_schema import AuthSchema, PageResultSchema
from app.core.exceptions import CustomException
@@ -71,25 +72,36 @@ class JobService:
state = SchedulerUtil.get_scheduler_state()
is_running = SchedulerUtil.is_running()
jobs = SchedulerUtil.get_jobs()
# 可观测性（t15）：把"已注册但当前不可执行"的任务显式打出并计数，
# 运维不必逐个点开才知道开关调整/处理器改名后哪些任务静默失效
unschedulable = SchedulerUtil.log_unschedulable_jobs(context="调度器状态查询")
return SchedulerStatusSchema(
status=JobService._SCHEDULER_STATE_MAP.get(state, "未知"),
is_running=is_running,
job_count=len(jobs),
code_exec_enabled=settings.SCHEDULER_ALLOW_CODE_EXEC,
unschedulable_count=len(unschedulable),
)
@staticmethod
def get_scheduler_jobs() -> list[SchedulerJobSchema]:
jobs = SchedulerUtil.get_jobs()
return [
SchedulerJobSchema(
id=job.id,
name=job.name,
trigger=str(job.trigger),
next_run_time=str(job.next_run_time) if job.next_run_time else None,
status=SchedulerUtil.get_job_status(job_id=job.id),
items: list[SchedulerJobSchema] = []
for job in jobs:
schedulable, reason = SchedulerUtil.describe_schedulability(SchedulerUtil.job_code_block(job))
items.append(
SchedulerJobSchema(
id=job.id,
name=job.name,
trigger=str(job.trigger),
# pending/未排程任务没有 next_run_time 属性（APScheduler 只在真正排程后设置），必须 getattr 兜底
next_run_time=str(job.next_run_time) if getattr(job, "next_run_time", None) else None,
status=SchedulerUtil.get_job_status(job_id=job.id),
schedulable=schedulable,
unschedulable_reason=None if schedulable else reason,
),
)
for job in jobs
]
return items
# ─── 调度器与任务运维（controller 唯一入口，禁止直调 SchedulerUtil）───
backend/app/modules/task/cronjob/node/schema.py
+13 -1
View File
@@ -17,7 +17,14 @@ class NodeCreateSchema(BaseModel):
"""
name: str = Field(..., max_length=64, description="任务名称")
func: str | None = Field(default=None, description="代码块")
func: str | None = Field(
default=None,
description=(
"任务函数：两种写法——① 内置函数型 `builtin:<模块名>[.<函数名>]`（白名单解析后直接调用，不经 exec，"
"不受 SCHEDULER_ALLOW_CODE_EXEC 限制）；② 原始代码块（需定义 handler(*args, **kwargs)，等同服务器代码执行权限，"
"仅在 SCHEDULER_ALLOW_CODE_EXEC=true 时允许）"
),
)
args: str | None = Field(default=None, description="位置参数")
kwargs: str | None = Field(default=None, description="关键字参数")
coalesce: bool | None = Field(default=False, description="是否合并运行:是否在多个运行时间到期时仅运行作业一次")
@@ -102,6 +109,11 @@ class NodeOutSchema(NodeCreateSchema, BaseSchema, UserBySchema):
next_run_time: str | None = Field(default=None, description="下次运行时间")
last_run_time: str | None = Field(default=None, description="最近一次运行时间")
last_run_status: int | None = Field(default=None, description="最近一次运行状态(2:成功 3:失败)")
schedulable: bool | None = Field(
default=None,
description="当前是否可调度；false 表示「已启用也不会执行」（如代码块被禁用、内置处理器不存在、未配置函数）",
)
unschedulable_reason: str | None = Field(default=None, description="不可调度原因（含修复建议）")
model_config = ConfigDict(from_attributes=True)
backend/app/modules/task/cronjob/node/service.py
+55 -6
View File
@@ -11,7 +11,6 @@ from croniter import croniter
from sqlalchemy import false, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.config.setting import settings
from app.core.ap_scheduler import SchedulerUtil, scheduler
from app.core.base_schema import AuthSchema, PageResultSchema
from app.core.exceptions import CustomException
@@ -179,7 +178,17 @@ class NodeService:
node_id = str(item.id) if item.id is not None else ""
job = jobs.get(node_id)
if job:
item.next_run_time = str(job.next_run_time) if job.next_run_time else None
item.next_run_time = str(job.next_run_time) if getattr(job, "next_run_time", None) else None
# 可调度状态（t15）：区分「已启用且可执行」与「已启用但当前不会执行」，
# 并把后者以 WARN 打出（带节点标识），避免升级/开关调整后的静默失效
schedulable, reason = SchedulerUtil.describe_schedulability(item.func)
item.schedulable = schedulable
item.unschedulable_reason = None if schedulable else reason
if not schedulable and item.status == 0:
_warn_unschedulable_node(node_id=node_id, name=item.name, reason=reason)
elif schedulable:
_clear_unschedulable_warning(node_id)
stmt = (
select(JobModel)
.where(JobModel.job_id == node_id, JobModel.is_deleted == false())
@@ -192,6 +201,41 @@ class NodeService:
item.last_run_status = row.status
# 「已启用但不可调度」的告警去重：同一节点 + 同一原因只告警一次，
# 避免列表接口被反复调用时刷屏；节点恢复可调度时清除记录（见 _enrich_runtime）。
_WARNED_UNSCHEDULABLE: dict[str, str] = {}
def _warn_unschedulable_node(*, node_id: str, name: str | None, reason: str | None) -> None:
"""对「已启用但当前不可调度」的节点打一次 WARN（带节点标识与原因）。
参数:
- node_id (str): 节点 ID（调度器 job id）。
- name (str | None): 节点名称。
- reason (str | None): 不可调度原因。
返回:
- None
"""
text = reason or ""
if _WARNED_UNSCHEDULABLE.get(node_id) == text:
return
_WARNED_UNSCHEDULABLE[node_id] = text
logger.warning(f"节点任务不可调度（已启用但不会执行）: id={node_id} name={name!r} 原因={reason}")
def _clear_unschedulable_warning(node_id: str) -> None:
"""节点恢复可调度后清除其告警去重记录（下次再失效仍会告警）。
参数:
- node_id (str): 节点 ID。
返回:
- None
"""
_WARNED_UNSCHEDULABLE.pop(node_id, None)
# ── NodeModel 封装的调度操作（模块级函数，供 service / 启动自愈复用）────
@@ -281,8 +325,11 @@ def _add_job_with_trigger(
code_block = job_info.func
if not code_block or not code_block.strip():
raise ValueError("任务代码块不能为空")
if not settings.SCHEDULER_ALLOW_CODE_EXEC:
raise CustomException(msg="服务已禁用定时任务代码执行（SCHEDULER_ALLOW_CODE_EXEC=False），无法注册代码块任务")
# 可调度性判定（单一事实来源）：builtin: 引用走白名单、不经 exec，因此**开关关闭时也允许注册**；
# 只有"原始代码块"才会被 SCHEDULER_ALLOW_CODE_EXEC=False 拦住，并在提示里给出 builtin 替代写法。
schedulable, reason = SchedulerUtil.describe_schedulability(code_block)
if not schedulable:
raise CustomException(msg=reason)
jobstore = job_info.jobstore or "default"
executor = job_info.executor or "threadpool"
@@ -368,8 +415,10 @@ def run_node_once(node: NodeModel) -> str:
"""
if not node.func or not node.func.strip():
raise ValueError("任务代码块不能为空")
if not settings.SCHEDULER_ALLOW_CODE_EXEC:
raise CustomException(msg="服务已禁用定时任务代码执行（SCHEDULER_ALLOW_CODE_EXEC=False），无法执行代码块任务")
# 手动执行同样按可调度性判定：builtin 引用可直接执行；原始代码块在开关关闭时给出替代写法
schedulable, reason = SchedulerUtil.describe_schedulability(node.func)
if not schedulable:
raise CustomException(msg=reason)
temp_job_id = f"{node.id}{_MANUAL_JOB_PREFIX}{datetime.now():%Y%m%d%H%M%S}"
trigger = DateTrigger(run_date=datetime.now() + timedelta(seconds=0.1), timezone="Asia/Shanghai")
_add_job_with_trigger(node, trigger, job_id_override=temp_job_id, name_suffix=" - 手动执行")
backend/app/modules/task/storage/core/base.py
+3 -3
View File
@@ -80,7 +80,7 @@ def decrypt_password(cipher: str | None) -> str:
try:
return CryptoUtil.decrypt(cipher)
except CustomException as e:
raise CustomException(msg=f"存储源密码解密失败：{e!s}")
raise CustomException(msg=f"存储源密码解密失败：{e!s}", status_code=500)
# 文件系统类协议与对象存储类协议的划分：仅用于文档说明与配置元数据，
@@ -322,7 +322,7 @@ class BaseStorageAdapter(ABC):
with ThreadPoolExecutor(max_workers=workers) as pool:
list(pool.map(_run, tasks))
if errors:
raise CustomException(msg=f"批量上传失败 {len(errors)}/{len(tasks)} 个文件，首个错误: {errors[0]!s}")
raise CustomException(msg=f"批量上传失败 {len(errors)}/{len(tasks)} 个文件，首个错误: {errors[0]!s}", status_code=500)
return len(tasks)
def _sync_download_dir(self, remote_dir: str, local_dir: str, concurrency: int) -> int:
@@ -352,7 +352,7 @@ class BaseStorageAdapter(ABC):
with ThreadPoolExecutor(max_workers=workers) as pool:
list(pool.map(_run, tasks))
if errors:
raise CustomException(msg=f"批量下载失败 {len(errors)}/{len(tasks)} 个文件，首个错误: {errors[0]!s}")
raise CustomException(msg=f"批量下载失败 {len(errors)}/{len(tasks)} 个文件，首个错误: {errors[0]!s}", status_code=500)
return len(tasks)
@staticmethod
backend/app/modules/task/storage/core/cos_adapter.py
+12 -12
View File
@@ -96,7 +96,7 @@ class CosStorageAdapter(BaseStorageAdapter):
EnableMD5=self._adv.enable_md5,
)
except Exception as e:
raise CustomException(msg=f"COS 上传失败: {e!s}")
raise CustomException(msg=f"COS 上传失败: {e!s}", status_code=500)
return remote_path
def _sync_download(self, remote_path: str, local_path: str) -> str:
@@ -107,7 +107,7 @@ class CosStorageAdapter(BaseStorageAdapter):
DestFilePath=local_path,
)
except Exception as e:
raise CustomException(msg=f"COS 下载失败: {e!s}")
raise CustomException(msg=f"COS 下载失败: {e!s}", status_code=500)
return local_path
def _sync_delete(self, remote_path: str) -> None:
@@ -122,7 +122,7 @@ class CosStorageAdapter(BaseStorageAdapter):
try:
self.client.delete_object(Bucket=self._require_bucket(), Key=remote_path)
except Exception as e:
raise CustomException(msg=f"COS 删除失败: {e!s}")
raise CustomException(msg=f"COS 删除失败: {e!s}", status_code=500)
def _sync_exists(self, remote_path: str) -> bool:
try:
@@ -181,7 +181,7 @@ class CosStorageAdapter(BaseStorageAdapter):
break
return result
except Exception as e:
raise CustomException(msg=f"COS 列表失败: {e!s}")
raise CustomException(msg=f"COS 列表失败: {e!s}", status_code=500)
def _sync_list_page(self, prefix: str, page_size: int, cursor: str | None) -> StoragePage:
"""游标分页：单次 SDK 请求只拉一页（COS Marker），翻页经前端回传游标。"""
@@ -250,7 +250,7 @@ class CosStorageAdapter(BaseStorageAdapter):
next_cursor=next_cursor or None,
)
except Exception as e:
raise CustomException(msg=f"COS 列表失败: {e!s}")
raise CustomException(msg=f"COS 列表失败: {e!s}", status_code=500)
def _sync_list_buckets(self) -> list[str]:
"""列出账号下全部存储桶。"""
@@ -259,7 +259,7 @@ class CosStorageAdapter(BaseStorageAdapter):
buckets = (resp.get("Buckets") or {}).get("Bucket") or []
return [b.get("Name", "") for b in buckets if b.get("Name")]
except Exception as e:
raise CustomException(msg=f"COS 桶列表失败: {e!s}")
raise CustomException(msg=f"COS 桶列表失败: {e!s}", status_code=500)
def _sync_get_url(self, remote_path: str, expire: int) -> str:
"""生成预签名 URL
@@ -280,7 +280,7 @@ class CosStorageAdapter(BaseStorageAdapter):
Expired=expire,
)
except Exception as e:
raise CustomException(msg=f"COS 生成预签名 URL 失败: {e!s}")
raise CustomException(msg=f"COS 生成预签名 URL 失败: {e!s}", status_code=500)
# ── 目录操作（对象存储以 key/ 占位对象模拟目录）──────────────────
@@ -311,13 +311,13 @@ class CosStorageAdapter(BaseStorageAdapter):
try:
self.client.put_object(Bucket=self._require_bucket(), Key=remote_dir.rstrip("/") + "/", Body=b"")
except Exception as e:
raise CustomException(msg=f"COS 创建目录失败: {e!s}")
raise CustomException(msg=f"COS 创建目录失败: {e!s}", status_code=500)
def _sync_rmdir(self, remote_dir: str) -> None:
try:
self.client.delete_object(Bucket=self._require_bucket(), Key=remote_dir.rstrip("/") + "/")
except Exception as e:
raise CustomException(msg=f"COS 删除目录失败: {e!s}")
raise CustomException(msg=f"COS 删除目录失败: {e!s}", status_code=500)
def _sync_copy(self, src: str, dst: str) -> None:
"""复制：目录走基类递归实现；文件用服务端 copy_object。"""
@@ -331,7 +331,7 @@ class CosStorageAdapter(BaseStorageAdapter):
CopySource={"Bucket": self._require_bucket(), "Key": src},
)
except Exception as e:
raise CustomException(msg=f"COS 复制失败: {e!s}")
raise CustomException(msg=f"COS 复制失败: {e!s}", status_code=500)
def _sync_rename(self, src: str, dst: str) -> None:
"""重命名/移动：目录先复制后删除；文件 copy_object 后删源。"""
@@ -345,7 +345,7 @@ class CosStorageAdapter(BaseStorageAdapter):
try:
keys = self._list_all_keys(prefix)
except Exception as e:
raise CustomException(msg=f"COS 递归列表失败: {e!s}")
raise CustomException(msg=f"COS 递归列表失败: {e!s}", status_code=500)
return self._entries_from_keys(keys)
def _sync_delete_dir(self, remote_dir: str) -> None:
@@ -361,4 +361,4 @@ class CosStorageAdapter(BaseStorageAdapter):
Delete={"Objects": [{"Key": k} for k in keys[i : i + 1000]], "Quiet": True},
)
except Exception as e:
raise CustomException(msg=f"COS 递归删除失败: {e!s}")
raise CustomException(msg=f"COS 递归删除失败: {e!s}", status_code=500)
backend/app/modules/task/storage/core/ftp_ftps_adapter.py
+8 -8
View File
@@ -101,7 +101,7 @@ class FtpStorageAdapter(BaseStorageAdapter):
with open(local_path, "rb") as f:
client.storbinary(f"STOR {remote_path}", f)
except Exception as e:
raise CustomException(msg=f"FTP 上传失败: {e!s}")
raise CustomException(msg=f"FTP 上传失败: {e!s}", status_code=500)
finally:
self._close_client(client)
return remote_path
@@ -123,7 +123,7 @@ class FtpStorageAdapter(BaseStorageAdapter):
with open(local_path, "wb") as f:
client.retrbinary(f"RETR {remote_path}", f.write)
except Exception as e:
raise CustomException(msg=f"FTP 下载失败: {e!s}")
raise CustomException(msg=f"FTP 下载失败: {e!s}", status_code=500)
finally:
self._close_client(client)
return local_path
@@ -133,7 +133,7 @@ class FtpStorageAdapter(BaseStorageAdapter):
try:
client.delete(remote_path)
except Exception as e:
raise CustomException(msg=f"FTP 删除失败: {e!s}")
raise CustomException(msg=f"FTP 删除失败: {e!s}", status_code=500)
finally:
self._close_client(client)
@@ -182,7 +182,7 @@ class FtpStorageAdapter(BaseStorageAdapter):
)
return result
except Exception as e:
raise CustomException(msg=f"FTP 列表失败: {e!s}")
raise CustomException(msg=f"FTP 列表失败: {e!s}", status_code=500)
finally:
self._close_client(client)
@@ -200,7 +200,7 @@ class FtpStorageAdapter(BaseStorageAdapter):
except ftplib.error_perm:
pass # 目录已存在时跳过（部分服务器返回 550）
except Exception as e:
raise CustomException(msg=f"FTP 创建目录失败: {e!s}")
raise CustomException(msg=f"FTP 创建目录失败: {e!s}", status_code=500)
finally:
self._close_client(client)
@@ -209,7 +209,7 @@ class FtpStorageAdapter(BaseStorageAdapter):
try:
client.rmd(remote_dir)
except Exception as e:
raise CustomException(msg=f"FTP 删除目录失败: {e!s}")
raise CustomException(msg=f"FTP 删除目录失败: {e!s}", status_code=500)
finally:
self._close_client(client)
@@ -218,7 +218,7 @@ class FtpStorageAdapter(BaseStorageAdapter):
try:
client.rename(src, dst)
except Exception as e:
raise CustomException(msg=f"FTP 重命名失败: {e!s}")
raise CustomException(msg=f"FTP 重命名失败: {e!s}", status_code=500)
finally:
self._close_client(client)
@@ -248,6 +248,6 @@ class FtpStorageAdapter(BaseStorageAdapter):
finally:
os.remove(tmp_path)
except Exception as e:
raise CustomException(msg=f"FTP 复制失败: {e!s}")
raise CustomException(msg=f"FTP 复制失败: {e!s}", status_code=500)
finally:
self._close_client(client)
backend/app/modules/task/storage/core/local_adapter.py
+8 -8
View File
@@ -37,7 +37,7 @@ class LocalStorageAdapter(BaseStorageAdapter):
os.makedirs(os.path.dirname(dest), exist_ok=True)
shutil.copy2(local_path, dest)
except Exception as e:
raise CustomException(msg=f"本地存储上传失败: {e!s}")
raise CustomException(msg=f"本地存储上传失败: {e!s}", status_code=500)
return remote_path
def _sync_download(self, remote_path: str, local_path: str) -> str:
@@ -45,7 +45,7 @@ class LocalStorageAdapter(BaseStorageAdapter):
try:
shutil.copy2(src, local_path)
except Exception as e:
raise CustomException(msg=f"本地存储下载失败: {e!s}")
raise CustomException(msg=f"本地存储下载失败: {e!s}", status_code=500)
return local_path
def _sync_delete(self, remote_path: str) -> None:
@@ -56,7 +56,7 @@ class LocalStorageAdapter(BaseStorageAdapter):
elif os.path.exists(target):
os.remove(target)
except Exception as e:
raise CustomException(msg=f"本地存储删除失败: {e!s}")
raise CustomException(msg=f"本地存储删除失败: {e!s}", status_code=500)
def _sync_exists(self, remote_path: str) -> bool:
return os.path.exists(self._abs_path(remote_path))
@@ -66,7 +66,7 @@ class LocalStorageAdapter(BaseStorageAdapter):
try:
entries = os.scandir(base_dir)
except OSError as e:
raise CustomException(msg=f"本地存储列表失败: {e!s}")
raise CustomException(msg=f"本地存储列表失败: {e!s}", status_code=500)
result: list[StorageObject] = []
for entry in entries:
@@ -93,20 +93,20 @@ class LocalStorageAdapter(BaseStorageAdapter):
try:
os.makedirs(self._abs_path(remote_dir), exist_ok=True)
except Exception as e:
raise CustomException(msg=f"本地存储创建目录失败: {e!s}")
raise CustomException(msg=f"本地存储创建目录失败: {e!s}", status_code=500)
def _sync_rmdir(self, remote_dir: str) -> None:
try:
os.rmdir(self._abs_path(remote_dir))
except Exception as e:
raise CustomException(msg=f"本地存储删除目录失败: {e!s}")
raise CustomException(msg=f"本地存储删除目录失败: {e!s}", status_code=500)
def _sync_rename(self, src: str, dst: str) -> None:
try:
os.makedirs(os.path.dirname(self._abs_path(dst)), exist_ok=True)
shutil.move(self._abs_path(src), self._abs_path(dst))
except Exception as e:
raise CustomException(msg=f"本地存储重命名失败: {e!s}")
raise CustomException(msg=f"本地存储重命名失败: {e!s}", status_code=500)
def _sync_copy(self, src: str, dst: str) -> None:
src_abs, dst_abs = self._abs_path(src), self._abs_path(dst)
@@ -117,4 +117,4 @@ class LocalStorageAdapter(BaseStorageAdapter):
os.makedirs(os.path.dirname(dst_abs), exist_ok=True)
shutil.copy2(src_abs, dst_abs)
except Exception as e:
raise CustomException(msg=f"本地存储复制失败: {e!s}")
raise CustomException(msg=f"本地存储复制失败: {e!s}", status_code=500)
backend/app/modules/task/storage/core/obs_adapter.py
+24 -24
View File
@@ -176,7 +176,7 @@ class ObsStorageAdapter(BaseStorageAdapter):
content=f,
)
if not self._is_ok(resp):
raise CustomException(msg=f"OBS 上传失败: {self._error_desc(resp)}")
raise CustomException(msg=f"OBS 上传失败: {self._error_desc(resp)}", status_code=500)
return remote_path
part_size, concurrency, _ = self._multipart_settings()
resp = self.client.uploadFile(
@@ -188,11 +188,11 @@ class ObsStorageAdapter(BaseStorageAdapter):
enableCheckpoint=True,
)
if not self._is_ok(resp):
raise CustomException(msg=f"OBS 上传失败: {self._error_desc(resp)}")
raise CustomException(msg=f"OBS 上传失败: {self._error_desc(resp)}", status_code=500)
except CustomException:
raise
except Exception as e:
raise CustomException(msg=f"OBS 上传失败: {e!s}")
raise CustomException(msg=f"OBS 上传失败: {e!s}", status_code=500)
return remote_path
def _sync_download(self, remote_path: str, local_path: str) -> str:
@@ -207,22 +207,22 @@ class ObsStorageAdapter(BaseStorageAdapter):
enableCheckpoint=True,
)
if not self._is_ok(resp):
raise CustomException(msg=f"OBS 下载失败: {self._error_desc(resp)}")
raise CustomException(msg=f"OBS 下载失败: {self._error_desc(resp)}", status_code=500)
except CustomException:
raise
except Exception as e:
raise CustomException(msg=f"OBS 下载失败: {e!s}")
raise CustomException(msg=f"OBS 下载失败: {e!s}", status_code=500)
return local_path
def _sync_delete(self, remote_path: str) -> None:
try:
resp = self.client.deleteObject(bucketName=self._require_bucket(), objectKey=remote_path)
if not self._is_ok(resp):
raise CustomException(msg=f"OBS 删除失败: {self._error_desc(resp)}")
raise CustomException(msg=f"OBS 删除失败: {self._error_desc(resp)}", status_code=500)
except CustomException:
raise
except Exception as e:
raise CustomException(msg=f"OBS 删除失败: {e!s}")
raise CustomException(msg=f"OBS 删除失败: {e!s}", status_code=500)
def _sync_exists(self, remote_path: str) -> bool:
try:
@@ -255,7 +255,7 @@ class ObsStorageAdapter(BaseStorageAdapter):
kwargs["marker"] = marker
resp = self.client.listObjects(**kwargs)
if not self._is_ok(resp):
raise CustomException(msg=f"OBS 列表失败: {self._error_desc(resp)}")
raise CustomException(msg=f"OBS 列表失败: {self._error_desc(resp)}", status_code=500)
body = getattr(resp, "body", None)
for common in getattr(body, "commonPrefixs", None) or []:
raw_key = (getattr(common, "prefix", "") or "").rstrip("/")
@@ -292,7 +292,7 @@ class ObsStorageAdapter(BaseStorageAdapter):
except CustomException:
raise
except Exception as e:
raise CustomException(msg=f"OBS 列表失败: {e!s}")
raise CustomException(msg=f"OBS 列表失败: {e!s}", status_code=500)
def _sync_list_page(self, prefix: str, page_size: int, cursor: str | None) -> StoragePage:
"""游标分页：单次 SDK 请求只拉一页（OBS marker），翻页经前端回传游标。"""
@@ -307,7 +307,7 @@ class ObsStorageAdapter(BaseStorageAdapter):
kwargs["marker"] = cursor
resp = self.client.listObjects(**kwargs)
if not self._is_ok(resp):
raise CustomException(msg=f"OBS 列表失败: {self._error_desc(resp)}")
raise CustomException(msg=f"OBS 列表失败: {self._error_desc(resp)}", status_code=500)
body = getattr(resp, "body", None)
truncated = bool(getattr(body, "is_truncated", None))
next_cursor = getattr(body, "next_marker", None) or None
@@ -339,28 +339,28 @@ class ObsStorageAdapter(BaseStorageAdapter):
except CustomException:
raise
except Exception as e:
raise CustomException(msg=f"OBS 列表失败: {e!s}")
raise CustomException(msg=f"OBS 列表失败: {e!s}", status_code=500)
def _sync_list_buckets(self) -> list[str]:
"""列出账号下全部存储桶。"""
try:
resp = self.client.listBuckets()
if not self._is_ok(resp):
raise CustomException(msg=f"OBS 桶列表失败: {self._error_desc(resp)}")
raise CustomException(msg=f"OBS 桶列表失败: {self._error_desc(resp)}", status_code=500)
body = getattr(resp, "body", None)
buckets = getattr(body, "buckets", None) or []
return [getattr(b, "name", "") or "" for b in buckets if getattr(b, "name", "")]
except CustomException:
raise
except Exception as e:
raise CustomException(msg=f"OBS 桶列表失败: {e!s}")
raise CustomException(msg=f"OBS 桶列表失败: {e!s}", status_code=500)
def _sync_get_url(self, remote_path: str, expire: int) -> str:
try:
resp = self.client.createSignedUrl("GET", bucketName=self._require_bucket(), objectKey=remote_path, expires=expire)
return getattr(resp, "signedUrl", "")
except Exception as e:
raise CustomException(msg=f"OBS 生成预签名 URL 失败: {e!s}")
raise CustomException(msg=f"OBS 生成预签名 URL 失败: {e!s}", status_code=500)
# ── 目录操作（对象存储以 key/ 占位对象模拟目录）──────────────────
@@ -374,7 +374,7 @@ class ObsStorageAdapter(BaseStorageAdapter):
kwargs["marker"] = marker
resp = self.client.listObjects(**kwargs)
if not self._is_ok(resp):
raise CustomException(msg=f"OBS 列举对象失败: {self._error_desc(resp)}")
raise CustomException(msg=f"OBS 列举对象失败: {self._error_desc(resp)}", status_code=500)
body = getattr(resp, "body", None)
for obj in getattr(body, "contents", None) or []:
key = getattr(obj, "key", "")
@@ -403,21 +403,21 @@ class ObsStorageAdapter(BaseStorageAdapter):
try:
resp = self.client.putContent(bucketName=self._require_bucket(), objectKey=remote_dir.rstrip("/") + "/", content=b"")
if not self._is_ok(resp):
raise CustomException(msg=f"OBS 创建目录失败: {self._error_desc(resp)}")
raise CustomException(msg=f"OBS 创建目录失败: {self._error_desc(resp)}", status_code=500)
except CustomException:
raise
except Exception as e:
raise CustomException(msg=f"OBS 创建目录失败: {e!s}")
raise CustomException(msg=f"OBS 创建目录失败: {e!s}", status_code=500)
def _sync_rmdir(self, remote_dir: str) -> None:
try:
resp = self.client.deleteObject(bucketName=self._require_bucket(), objectKey=remote_dir.rstrip("/") + "/")
if not self._is_ok(resp):
raise CustomException(msg=f"OBS 删除目录失败: {self._error_desc(resp)}")
raise CustomException(msg=f"OBS 删除目录失败: {self._error_desc(resp)}", status_code=500)
except CustomException:
raise
except Exception as e:
raise CustomException(msg=f"OBS 删除目录失败: {e!s}")
raise CustomException(msg=f"OBS 删除目录失败: {e!s}", status_code=500)
def _sync_copy(self, src: str, dst: str) -> None:
"""复制：目录走基类递归实现；文件用服务端 copyObject。"""
@@ -432,11 +432,11 @@ class ObsStorageAdapter(BaseStorageAdapter):
destObjectKey=dst,
)
if not self._is_ok(resp):
raise CustomException(msg=f"OBS 复制失败: {self._error_desc(resp)}")
raise CustomException(msg=f"OBS 复制失败: {self._error_desc(resp)}", status_code=500)
except CustomException:
raise
except Exception as e:
raise CustomException(msg=f"OBS 复制失败: {e!s}")
raise CustomException(msg=f"OBS 复制失败: {e!s}", status_code=500)
def _sync_rename(self, src: str, dst: str) -> None:
"""重命名/移动：优先 OBS 原生 renameFile（支持目录级），失败回退先复制后删除。"""
@@ -456,7 +456,7 @@ class ObsStorageAdapter(BaseStorageAdapter):
try:
keys = self._list_all_keys(prefix)
except Exception as e:
raise CustomException(msg=f"OBS 递归列表失败: {e!s}")
raise CustomException(msg=f"OBS 递归列表失败: {e!s}", status_code=500)
return self._entries_from_keys(keys)
def _sync_delete_dir(self, remote_dir: str) -> None:
@@ -473,11 +473,11 @@ class ObsStorageAdapter(BaseStorageAdapter):
deleteObjectsRequest={"quiet": True, "objects": [{"key": k} for k in batch]},
)
if not self._is_ok(resp):
raise CustomException(msg=f"OBS 递归删除失败: {self._error_desc(resp)}")
raise CustomException(msg=f"OBS 递归删除失败: {self._error_desc(resp)}", status_code=500)
except CustomException:
raise
except Exception as e:
raise CustomException(msg=f"OBS 递归删除失败: {e!s}")
raise CustomException(msg=f"OBS 递归删除失败: {e!s}", status_code=500)
def _sync_close(self) -> None:
"""关闭 OBS 客户端连接。"""
backend/app/modules/task/storage/core/oss_adapter.py
+12 -12
View File
@@ -74,7 +74,7 @@ class OssStorageAdapter(BaseStorageAdapter):
parallel_num=concurrency,
)
except Exception as e:
raise CustomException(msg=f"OSS 上传失败: {e!s}")
raise CustomException(msg=f"OSS 上传失败: {e!s}", status_code=500)
return remote_path
def _sync_download(self, remote_path: str, local_path: str) -> str:
@@ -84,14 +84,14 @@ class OssStorageAdapter(BaseStorageAdapter):
local_path,
)
except Exception as e:
raise CustomException(msg=f"OSS 下载失败: {e!s}")
raise CustomException(msg=f"OSS 下载失败: {e!s}", status_code=500)
return local_path
def _sync_delete(self, remote_path: str) -> None:
try:
self.client.delete_object(oss.DeleteObjectRequest(bucket=self.bucket_name, key=remote_path))
except Exception as e:
raise CustomException(msg=f"OSS 删除失败: {e!s}")
raise CustomException(msg=f"OSS 删除失败: {e!s}", status_code=500)
def _sync_exists(self, remote_path: str) -> bool:
try:
@@ -140,7 +140,7 @@ class OssStorageAdapter(BaseStorageAdapter):
)
return result
except Exception as e:
raise CustomException(msg=f"OSS 列表失败: {e!s}")
raise CustomException(msg=f"OSS 列表失败: {e!s}", status_code=500)
def _sync_list_page(self, prefix: str, page_size: int, cursor: str | None) -> StoragePage:
"""游标分页：单次 SDK 请求只拉一页（OSS continuation_token），翻页经前端回传游标。"""
@@ -178,7 +178,7 @@ class OssStorageAdapter(BaseStorageAdapter):
)
return StoragePage(items=items, has_next=truncated, next_cursor=next_cursor)
except Exception as e:
raise CustomException(msg=f"OSS 列表失败: {e!s}")
raise CustomException(msg=f"OSS 列表失败: {e!s}", status_code=500)
def _sync_list_buckets(self) -> list[str]:
"""列出账号下全部存储桶。"""
@@ -191,7 +191,7 @@ class OssStorageAdapter(BaseStorageAdapter):
result.append(b.name)
return result
except Exception as e:
raise CustomException(msg=f"OSS 桶列表失败: {e!s}")
raise CustomException(msg=f"OSS 桶列表失败: {e!s}", status_code=500)
def _sync_get_url(self, remote_path: str, expire: int) -> str | None:
try:
@@ -201,7 +201,7 @@ class OssStorageAdapter(BaseStorageAdapter):
)
return result.url
except Exception as e:
raise CustomException(msg=f"OSS 生成预签名 URL 失败: {e!s}")
raise CustomException(msg=f"OSS 生成预签名 URL 失败: {e!s}", status_code=500)
# ── 目录操作（对象存储以 key/ 占位对象模拟目录）──────────────────
@@ -231,13 +231,13 @@ class OssStorageAdapter(BaseStorageAdapter):
try:
self.client.put_object(oss.PutObjectRequest(bucket=self.bucket_name, key=remote_dir.rstrip("/") + "/", body=b""))
except Exception as e:
raise CustomException(msg=f"OSS 创建目录失败: {e!s}")
raise CustomException(msg=f"OSS 创建目录失败: {e!s}", status_code=500)
def _sync_rmdir(self, remote_dir: str) -> None:
try:
self.client.delete_object(oss.DeleteObjectRequest(bucket=self.bucket_name, key=remote_dir.rstrip("/") + "/"))
except Exception as e:
raise CustomException(msg=f"OSS 删除目录失败: {e!s}")
raise CustomException(msg=f"OSS 删除目录失败: {e!s}", status_code=500)
def _sync_copy(self, src: str, dst: str) -> None:
"""复制：目录走基类递归实现；文件用服务端 copy_object。"""
@@ -249,7 +249,7 @@ class OssStorageAdapter(BaseStorageAdapter):
oss.CopyObjectRequest(bucket=self.bucket_name, key=dst, source_bucket=self.bucket_name, source_key=src)
)
except Exception as e:
raise CustomException(msg=f"OSS 复制失败: {e!s}")
raise CustomException(msg=f"OSS 复制失败: {e!s}", status_code=500)
def _sync_rename(self, src: str, dst: str) -> None:
"""重命名/移动：目录先复制后删除；文件 copy_object 后删源。"""
@@ -263,7 +263,7 @@ class OssStorageAdapter(BaseStorageAdapter):
try:
keys = self._list_all_keys(prefix)
except Exception as e:
raise CustomException(msg=f"OSS 递归列表失败: {e!s}")
raise CustomException(msg=f"OSS 递归列表失败: {e!s}", status_code=500)
return self._entries_from_keys(keys)
def _sync_delete_dir(self, remote_dir: str) -> None:
@@ -282,4 +282,4 @@ class OssStorageAdapter(BaseStorageAdapter):
)
)
except Exception as e:
raise CustomException(msg=f"OSS 递归删除失败: {e!s}")
raise CustomException(msg=f"OSS 递归删除失败: {e!s}", status_code=500)
backend/app/modules/task/storage/core/s3_adapter.py
+12 -12
View File
@@ -88,21 +88,21 @@ class S3StorageAdapter(BaseStorageAdapter):
)
self.client.upload_file(local_path, self._require_bucket(), remote_path, Config=transfer_config)
except Exception as e:
raise CustomException(msg=f"S3 上传失败: {e!s}")
raise CustomException(msg=f"S3 上传失败: {e!s}", status_code=500)
return remote_path
def _sync_download(self, remote_path: str, local_path: str) -> str:
try:
self.client.download_file(self._require_bucket(), remote_path, local_path)
except Exception as e:
raise CustomException(msg=f"S3 下载失败: {e!s}")
raise CustomException(msg=f"S3 下载失败: {e!s}", status_code=500)
return local_path
def _sync_delete(self, remote_path: str) -> None:
try:
self.client.delete_object(Bucket=self._require_bucket(), Key=remote_path)
except Exception as e:
raise CustomException(msg=f"S3 删除失败: {e!s}")
raise CustomException(msg=f"S3 删除失败: {e!s}", status_code=500)
def _sync_exists(self, remote_path: str) -> bool:
try:
@@ -163,7 +163,7 @@ class S3StorageAdapter(BaseStorageAdapter):
break
return result
except Exception as e:
raise CustomException(msg=f"S3 列表失败: {e!s}")
raise CustomException(msg=f"S3 列表失败: {e!s}", status_code=500)
def _sync_list_page(self, prefix: str, page_size: int, cursor: str | None) -> StoragePage:
"""游标分页：单次 SDK 请求只拉一页（S3 ContinuationToken），翻页经前端回传游标。"""
@@ -203,7 +203,7 @@ class S3StorageAdapter(BaseStorageAdapter):
next_cursor=resp.get("NextContinuationToken"),
)
except Exception as e:
raise CustomException(msg=f"S3 列表失败: {e!s}")
raise CustomException(msg=f"S3 列表失败: {e!s}", status_code=500)
def _sync_list_buckets(self) -> list[str]:
"""列出账号下全部存储桶。"""
@@ -211,7 +211,7 @@ class S3StorageAdapter(BaseStorageAdapter):
resp = self.client.list_buckets()
return [b.get("Name", "") for b in resp.get("Buckets", []) if b.get("Name")]
except Exception as e:
raise CustomException(msg=f"S3 桶列表失败: {e!s}")
raise CustomException(msg=f"S3 桶列表失败: {e!s}", status_code=500)
def _sync_get_url(self, remote_path: str, expire: int) -> str:
try:
@@ -221,7 +221,7 @@ class S3StorageAdapter(BaseStorageAdapter):
ExpiresIn=expire,
)
except Exception as e:
raise CustomException(msg=f"S3 生成预签名 URL 失败: {e!s}")
raise CustomException(msg=f"S3 生成预签名 URL 失败: {e!s}", status_code=500)
def _sync_close(self) -> None:
"""关闭 boto3 客户端连接。"""
@@ -253,13 +253,13 @@ class S3StorageAdapter(BaseStorageAdapter):
try:
self.client.put_object(Bucket=self._require_bucket(), Key=remote_dir.rstrip("/") + "/", Body=b"")
except Exception as e:
raise CustomException(msg=f"S3 创建目录失败: {e!s}")
raise CustomException(msg=f"S3 创建目录失败: {e!s}", status_code=500)
def _sync_rmdir(self, remote_dir: str) -> None:
try:
self.client.delete_object(Bucket=self._require_bucket(), Key=remote_dir.rstrip("/") + "/")
except Exception as e:
raise CustomException(msg=f"S3 删除目录失败: {e!s}")
raise CustomException(msg=f"S3 删除目录失败: {e!s}", status_code=500)
def _sync_copy(self, src: str, dst: str) -> None:
"""复制：目录走基类递归实现；文件用服务端 copy_object。"""
@@ -273,7 +273,7 @@ class S3StorageAdapter(BaseStorageAdapter):
Key=dst,
)
except Exception as e:
raise CustomException(msg=f"S3 复制失败: {e!s}")
raise CustomException(msg=f"S3 复制失败: {e!s}", status_code=500)
def _sync_rename(self, src: str, dst: str) -> None:
"""重命名/移动：目录先复制后删除；文件 copy_object 后删源。"""
@@ -287,7 +287,7 @@ class S3StorageAdapter(BaseStorageAdapter):
try:
keys = self._list_all_keys(prefix)
except Exception as e:
raise CustomException(msg=f"S3 递归列表失败: {e!s}")
raise CustomException(msg=f"S3 递归列表失败: {e!s}", status_code=500)
return self._entries_from_keys(keys)
def _sync_delete_dir(self, remote_dir: str) -> None:
@@ -303,4 +303,4 @@ class S3StorageAdapter(BaseStorageAdapter):
Delete={"Objects": [{"Key": k} for k in keys[i : i + 1000]], "Quiet": True},
)
except Exception as e:
raise CustomException(msg=f"S3 递归删除失败: {e!s}")
raise CustomException(msg=f"S3 递归删除失败: {e!s}", status_code=500)
backend/app/modules/task/storage/core/sftp_adapter.py
+8 -8
View File
@@ -118,7 +118,7 @@ class SftpStorageAdapter(BaseStorageAdapter):
self._ensure_remote_dir(client, dir_part)
client.put(local_path, remote_path)
except Exception as e:
raise CustomException(msg=f"SFTP 上传失败: {e!s}")
raise CustomException(msg=f"SFTP 上传失败: {e!s}", status_code=500)
return remote_path
def _sync_download(self, remote_path: str, local_path: str) -> str:
@@ -141,14 +141,14 @@ class SftpStorageAdapter(BaseStorageAdapter):
else:
client.get(remote_path, local_path)
except Exception as e:
raise CustomException(msg=f"SFTP 下载失败: {e!s}")
raise CustomException(msg=f"SFTP 下载失败: {e!s}", status_code=500)
return local_path
def _sync_delete(self, remote_path: str) -> None:
try:
self._connect().remove(remote_path)
except Exception as e:
raise CustomException(msg=f"SFTP 删除失败: {e!s}")
raise CustomException(msg=f"SFTP 删除失败: {e!s}", status_code=500)
def _sync_exists(self, remote_path: str) -> bool:
try:
@@ -162,7 +162,7 @@ class SftpStorageAdapter(BaseStorageAdapter):
# 空 prefix 表示浏览根目录：SFTP 用 "." 表示登录后的当前目录
attrs = self._connect().listdir_attr(prefix or ".")
except Exception as e:
raise CustomException(msg=f"SFTP 列表失败: {e!s}")
raise CustomException(msg=f"SFTP 列表失败: {e!s}", status_code=500)
result: list[StorageObject] = []
for attr in attrs:
result.append(
@@ -189,19 +189,19 @@ class SftpStorageAdapter(BaseStorageAdapter):
try:
self._ensure_remote_dir(self._connect(), remote_dir)
except Exception as e:
raise CustomException(msg=f"SFTP 创建目录失败: {e!s}")
raise CustomException(msg=f"SFTP 创建目录失败: {e!s}", status_code=500)
def _sync_rmdir(self, remote_dir: str) -> None:
try:
self._connect().rmdir(remote_dir)
except Exception as e:
raise CustomException(msg=f"SFTP 删除目录失败: {e!s}")
raise CustomException(msg=f"SFTP 删除目录失败: {e!s}", status_code=500)
def _sync_rename(self, src: str, dst: str) -> None:
try:
self._connect().posix_rename(src, dst)
except Exception as e:
raise CustomException(msg=f"SFTP 重命名失败: {e!s}")
raise CustomException(msg=f"SFTP 重命名失败: {e!s}", status_code=500)
def _sync_copy(self, src: str, dst: str) -> None:
"""复制：目录走基类递归实现；文件用通道流式复制（SFTP 无服务端复制）。"""
@@ -214,4 +214,4 @@ class SftpStorageAdapter(BaseStorageAdapter):
while chunk := rf.read(1024 * 1024):
wf.write(chunk)
except Exception as e:
raise CustomException(msg=f"SFTP 复制失败: {e!s}")
raise CustomException(msg=f"SFTP 复制失败: {e!s}", status_code=500)