feat: add storage module, internal chat and workflow enhancements

- 新增多协议存储模块(支持FTP/SFTP/S3/OBS/OSS/COS/本地存储),包含存储源管理、文件操作、传输任务功能
- 新增系统内部聊天模块,支持WebSocket实时通信与未读统计
- 优化工作流节点系统,支持内置业务节点并完善配置面板
- 修复健康检查接口路径与菜单角标显示逻辑
- 新增SQLite WAL模式与busy_timeout配置优化并发性能
- 更新依赖与文档,支持模块裁剪配置
This commit is contained in:
zhangtao
2026-08-19 23:20:39 +08:00
parent 11160b05e4
commit 001d1bdf4c
72 changed files with 7672 additions and 550 deletions
@@ -0,0 +1,348 @@
"""工作流内置业务节点
内置节点以“代码即配置”的方式将存储、中转、通知等模块能力封装为
开箱即用的工作流节点,用户无需编写 Python 代码即可在画布上串联
多个模块,形成通用业务闭环。
执行约定:
- 引擎在 ThreadPoolExecutor 线程中调用节点 handler(同步函数),
因此 handler 内部通过 asyncio.run() 执行异步业务逻辑;
- handler 统一接收 **kwargs,其中 upstream / variables 由引擎注入,
业务参数直接从 kwargs 中读取;
- 内置节点不依赖数据库中的节点类型记录,引擎执行时优先命中内置节点,
其次才查询自定义节点类型。
"""
from __future__ import annotations
import asyncio
import os
import tempfile
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
from app.api.v1.module_storage.core.base import StorageAdapterConfig
from app.api.v1.module_storage.core.constants import StorageProtocol
from app.api.v1.module_storage.core.encrypt import decrypt_password as decrypt_storage_password
from app.api.v1.module_storage.core.factory import StorageAdapterFactory
from app.api.v1.module_storage.source.crud import StorageSourceCRUD
from app.api.v1.module_system.notice.crud import NoticeCRUD
from app.core.base_schema import AuthSchema
from app.core.database import async_db_session
from app.core.logger import logger
@dataclass
class BuiltinNode:
"""内置业务节点定义"""
code: str
name: str
category: str
description: str
handler: Callable[..., Any]
args: str = ""
kwargs: str = "{}"
BUILTIN_NODES: dict[str, BuiltinNode] = {}
def builtin_node(*, code: str, name: str, category: str = "action", description: str = "", args: str = "", kwargs: str = "{}") -> Callable:
"""注册内置节点。"""
def deco(fn: Callable[..., Any]) -> Callable[..., Any]:
BUILTIN_NODES[code] = BuiltinNode(
code=code,
name=name,
category=category,
description=description,
handler=fn,
args=args,
kwargs=kwargs,
)
return fn
return deco
def get_builtin_node(code: str) -> BuiltinNode | None:
return BUILTIN_NODES.get(code)
def builtin_node_options() -> list[dict]:
"""内置节点选项(合并进工作流设计器的节点类型选择)。"""
return [
{
"id": None,
"code": n.code,
"name": n.name,
"category": n.category,
"description": n.description,
"args": n.args,
"kwargs": n.kwargs,
}
for n in BUILTIN_NODES.values()
]
def _run_async(coro: Any) -> Any:
"""在线程中运行协程(工作流引擎在 ThreadPoolExecutor 中执行节点,无事件循环)。"""
return asyncio.run(coro)
def _system_auth() -> AuthSchema:
"""系统级调用上下文(不填充审计人,避免伪造用户)。"""
return AuthSchema()
# ── 存储节点 ──────────────────────────────────────────────────────
async def _resolve_storage_config(session, source_id: int | None) -> StorageAdapterConfig:
"""解析存储源配置(密码解密),source_id 为空时取默认源/任一启用源。"""
crud = StorageSourceCRUD(_system_auth(), session)
if source_id:
source = await crud.get_or_404(id=source_id, msg="存储源不存在")
if source.status == 1:
raise ValueError(f"存储源「{source.name}」已停用")
else:
source = await crud.get(status=0, is_default=True)
if source is None:
source = await crud.get(status=0)
if source is None:
raise ValueError("未配置可用的存储源,请先在存储源管理中创建")
return StorageAdapterConfig(
protocol=StorageProtocol(source.protocol),
host=source.host,
port=source.port,
username=source.username,
password=decrypt_storage_password(source.password),
bucket=source.bucket,
endpoint=source.endpoint,
region=source.region,
path_prefix=source.path_prefix,
is_secure=source.is_secure,
implicit_tls=source.implicit_tls,
)
@builtin_node(
code="storage_upload",
name="上传文件到存储源",
description="将服务器本地文件上传到指定存储源,返回远端路径与访问URL。",
kwargs='{"local_path": "/tmp/a.txt", "remote_path": "inbox/a.txt"}',
)
def storage_upload(**kwargs: Any) -> dict:
local_path = kwargs.get("local_path")
remote_path = kwargs.get("remote_path")
if not local_path or not remote_path:
raise ValueError("storage_upload 需要 local_path 与 remote_path 参数")
return _run_async(_storage_upload(kwargs.get("source_id"), str(local_path), str(remote_path)))
async def _storage_upload(source_id: int | None, local_path: str, remote_path: str) -> dict:
async with async_db_session() as session:
config = await _resolve_storage_config(session, source_id)
adapter = StorageAdapterFactory.create(config)
try:
await adapter.upload(local_path, remote_path)
file_url = await adapter.get_url(remote_path)
finally:
await adapter.close()
logger.info("工作流节点 storage_upload 完成: remote={}", remote_path)
return {"file_path": remote_path, "file_url": file_url}
@builtin_node(
code="storage_download",
name="从存储源下载文件",
description="将存储源文件下载到服务器本地,返回本地路径与文件名。",
kwargs='{"remote_path": "inbox/a.txt"}',
)
def storage_download(**kwargs: Any) -> dict:
remote_path = kwargs.get("remote_path")
if not remote_path:
raise ValueError("storage_download 需要 remote_path 参数")
return _run_async(_storage_download(kwargs.get("source_id"), str(remote_path), kwargs.get("local_path")))
async def _storage_download(source_id: int | None, remote_path: str, local_path: str | None) -> dict:
async with async_db_session() as session:
config = await _resolve_storage_config(session, source_id)
extension = os.path.splitext(remote_path)[1]
fd, tmp_path = tempfile.mkstemp(suffix=extension)
os.close(fd)
adapter = StorageAdapterFactory.create(config)
try:
await adapter.download(remote_path, local_path or tmp_path)
except Exception:
os.unlink(tmp_path)
raise
finally:
await adapter.close()
target = local_path or tmp_path
logger.info("工作流节点 storage_download 完成: local={}", target)
return {"local_path": target, "file_name": os.path.basename(remote_path)}
@builtin_node(
code="storage_url",
name="获取文件访问URL",
description="获取存储源文件的访问URL(预签名,默认1小时有效)。",
kwargs='{"remote_path": "inbox/a.txt", "expire": 3600}',
)
def storage_url(**kwargs: Any) -> dict:
remote_path = kwargs.get("remote_path")
if not remote_path:
raise ValueError("storage_url 需要 remote_path 参数")
return _run_async(_storage_url(kwargs.get("source_id"), str(remote_path), int(kwargs.get("expire", 3600))))
async def _storage_url(source_id: int | None, remote_path: str, expire: int) -> dict:
async with async_db_session() as session:
config = await _resolve_storage_config(session, source_id)
adapter = StorageAdapterFactory.create(config)
try:
file_url = await adapter.get_url(remote_path, expire=expire)
finally:
await adapter.close()
return {"file_url": file_url}
@builtin_node(
code="storage_exists",
name="判断存储文件是否存在",
description="检查存储源中指定文件是否存在,返回布尔结果,可作为条件节点使用。",
category="condition",
kwargs='{"remote_path": "inbox/a.txt"}',
)
def storage_exists(**kwargs: Any) -> dict:
remote_path = kwargs.get("remote_path")
if not remote_path:
raise ValueError("storage_exists 需要 remote_path 参数")
return _run_async(_storage_exists(kwargs.get("source_id"), str(remote_path)))
async def _storage_exists(source_id: int | None, remote_path: str) -> dict:
async with async_db_session() as session:
config = await _resolve_storage_config(session, source_id)
adapter = StorageAdapterFactory.create(config)
try:
exists = await adapter.exists(remote_path)
finally:
await adapter.close()
return {"exists": exists}
@builtin_node(
code="storage_delete",
name="删除存储文件",
description="删除存储源中的指定文件。",
kwargs='{"remote_path": "inbox/a.txt"}',
)
def storage_delete(**kwargs: Any) -> dict:
remote_path = kwargs.get("remote_path")
if not remote_path:
raise ValueError("storage_delete 需要 remote_path 参数")
return _run_async(_storage_delete(kwargs.get("source_id"), str(remote_path)))
async def _storage_delete(source_id: int | None, remote_path: str) -> dict:
async with async_db_session() as session:
config = await _resolve_storage_config(session, source_id)
adapter = StorageAdapterFactory.create(config)
try:
await adapter.delete(remote_path)
finally:
await adapter.close()
return {"deleted": True, "file_path": remote_path}
# ── 通知节点 ──────────────────────────────────────────────────────
@builtin_node(
code="notice_send",
name="发送站内通知",
description="发送一条站内通知(默认已发布,类型为通知)。",
kwargs='{"notice_title": "工作流通知", "notice_content": "任务执行完成"}',
)
def notice_send(**kwargs: Any) -> dict:
title = kwargs.get("notice_title")
if not title:
raise ValueError("notice_send 需要 notice_title 参数")
notice_type = str(kwargs.get("notice_type", "1"))
status = int(kwargs.get("status", 1))
content = kwargs.get("notice_content")
return _run_async(_notice_send(str(title), str(content) if content is not None else None, notice_type, status))
async def _notice_send(title: str, content: str | None, notice_type: str, status: int) -> dict:
async with async_db_session() as session:
obj = await NoticeCRUD(_system_auth(), session).create(
{
"notice_title": title,
"notice_type": notice_type,
"notice_content": content,
"status": status,
}
)
logger.info("工作流节点 notice_send 完成: notice={}", obj.id)
return {"notice_id": obj.id}
# ── AI 节点 ──────────────────────────────────────────────────────
@builtin_node(
code="ai_chat",
name="AI 对话/摘要",
description="调用 AI 模型处理文本(需先在 AI 模块配置并启用模型),返回模型回复。",
kwargs='{"message": "请总结以下内容: ..."}',
)
def ai_chat(**kwargs: Any) -> dict:
message = kwargs.get("message")
if not message:
raise ValueError("ai_chat 需要 message 参数")
session_id = kwargs.get("session_id")
return _run_async(_ai_chat(str(message), str(session_id) if session_id else None))
async def _ai_chat(message: str, session_id: str | None) -> dict:
from app.api.v1.module_ai.chat.service import ChatService
service = ChatService(_system_auth())
result = await service.chat_non_stream(message, session_id)
logger.info("工作流节点 ai_chat 完成: session={}", result.get("session_id"))
return {"response": result.get("response"), "session_id": result.get("session_id")}
# ── 通用检查节点 ──────────────────────────────────────────────────
@builtin_node(
code="http_check",
name="HTTP 健康检查",
description="请求指定 URL,HTTP 2xx/3xx 视为健康,返回布尔结果,可作为条件节点使用。",
category="condition",
kwargs='{"url": "http://127.0.0.1:8000/api/v1/health", "timeout": 5}',
)
def http_check(**kwargs: Any) -> dict:
url = kwargs.get("url")
if not url:
raise ValueError("http_check 需要 url 参数")
return _run_async(_http_check(str(url), float(kwargs.get("timeout", 5))))
async def _http_check(url: str, timeout: float) -> dict:
import httpx
try:
async with httpx.AsyncClient(timeout=timeout) as client:
resp = await client.get(url)
healthy = 200 <= resp.status_code < 400
return {"healthy": healthy, "status_code": resp.status_code, "url": url}
except Exception as e:
return {"healthy": False, "status_code": None, "url": url, "error": str(e)}
@@ -72,18 +72,22 @@ def _topological_levels(nodes: list[dict], edges: list[dict]) -> list[list[dict]
def _execute_node(
vue_node_id: str,
node_type_code: str,
code_block: str,
tpl: dict[str, Any],
args_str: str | None,
kwargs_str: str | None,
upstream: dict[str, Any],
flow_variables: dict[str, Any],
) -> Any:
job_id = f"wfnode-{vue_node_id}"
args = _parse_args(args_str)
kw = _parse_kwargs(kwargs_str)
kw.setdefault("upstream", upstream)
kw.setdefault("variables", flow_variables)
return SchedulerUtil._task_wrapper(job_id, code_block, *args, **kw)
# 内置业务节点:直接调用 handler 函数,不走代码块 exec
handler = tpl.get("handler")
if handler is not None:
return handler(*args, **kw)
job_id = f"wfnode-{vue_node_id}"
return SchedulerUtil._task_wrapper(job_id, tpl["func"], *args, **kw)
def run_workflow_sync(
@@ -103,7 +107,7 @@ def run_workflow_sync(
nid = node["id"]
ntype = node.get("type") or ""
tpl = node_templates.get(ntype)
if not tpl or not tpl.get("func"):
if not tpl or (not tpl.get("func") and "handler" not in tpl):
raise ValueError(f"未知或未配置节点类型: {ntype}")
data = node.get("data") or {}
args_str = data.get("args") if data.get("args") is not None else tpl.get("args")
@@ -116,7 +120,7 @@ def run_workflow_sync(
_execute_node,
nid,
ntype,
tpl["func"],
tpl,
args_str,
kwargs_str,
upstream,
@@ -9,6 +9,7 @@ from app.utils.common_util import search_to_dict
from ..node_type.crud import WorkflowNodeTypeCRUD
from .crud import WorkflowCRUD
from .handlers.builtin_nodes import get_builtin_node
from .handlers.workflow_engine import run_workflow_sync, utc_now_iso, validate_workflow_graph
from .schema import (
WorkflowCreateSchema,
@@ -143,19 +144,30 @@ class WorkflowService:
codes_set = {n.get("type") for n in nodes if n.get("type")}
code_list = list(codes_set)
templates: dict[str, dict[str, Any]] = {}
type_objs = await WorkflowNodeTypeCRUD(self.auth, self.db).get_obj_list_crud(search={"code": ("in", code_list)})
type_map = {t.code: t for t in type_objs}
# 内置业务节点优先命中(不依赖数据库节点类型记录)
for code in codes_set:
node_type = type_map.get(code)
if not node_type:
raise CustomException(msg=f"节点类型未注册(请在「工作流节点类型」中维护,非定时任务节点): {code}")
if not node_type.func or not str(node_type.func).strip():
raise CustomException(msg=f"节点类型未配置 func 代码块: {code}")
templates[code] = {
"func": node_type.func,
"args": node_type.args,
"kwargs": node_type.kwargs,
}
builtin = get_builtin_node(code)
if builtin is not None:
templates[code] = {
"handler": builtin.handler,
"args": builtin.args,
"kwargs": builtin.kwargs,
}
pending_codes = [c for c in code_list if c not in templates]
if pending_codes:
type_objs = await WorkflowNodeTypeCRUD(self.auth, self.db).get_obj_list_crud(search={"code": ("in", pending_codes)})
type_map = {t.code: t for t in type_objs}
for code in pending_codes:
node_type = type_map.get(code)
if not node_type:
raise CustomException(msg=f"节点类型未注册(请在「工作流节点类型」中维护,非定时任务节点): {code}")
if not node_type.func or not str(node_type.func).strip():
raise CustomException(msg=f"节点类型未配置 func 代码块: {code}")
templates[code] = {
"func": node_type.func,
"args": node_type.args,
"kwargs": node_type.kwargs,
}
variables = body.variables or {}
start = utc_now_iso()
@@ -4,6 +4,7 @@ from app.core.base_schema import AuthSchema, PageResultSchema
from app.core.exceptions import CustomException
from app.utils.common_util import search_to_dict
from ..flows.handlers.builtin_nodes import builtin_node_options
from .crud import WorkflowNodeTypeCRUD
from .schema import (
WorkflowNodeTypeCreateSchema,
@@ -26,17 +27,20 @@ class WorkflowNodeTypeService:
async def get_options(self) -> list[dict]:
objs = await WorkflowNodeTypeCRUD(self.auth, self.db).list_active_options_crud()
return [
db_options = [
{
"id": o.id,
"code": o.code,
"name": o.name,
"category": o.category,
"description": o.description or "",
"args": o.args or "",
"kwargs": o.kwargs or "{}",
}
for o in objs
]
# 内置业务节点排在前面,用户可开箱即用地串联存储/中转/通知等模块
return builtin_node_options() + db_options
async def get_detail(self, id: int) -> WorkflowNodeTypeOutSchema:
obj = await WorkflowNodeTypeCRUD(self.auth, self.db).get_obj_by_id_crud(id=id)