From 3f472d1e895c599b11ad9bd369ece8ae9bf4c7a5 Mon Sep 17 00:00:00 2001 From: zhangtao <9480807882@qq.com> Date: Thu, 3 Sep 2026 21:55:31 +0800 Subject: [PATCH] =?UTF-8?q?refactor:=20=E9=87=8D=E6=9E=84=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E4=B8=8E=E5=B7=A5=E4=BD=9C=E6=B5=81=E6=A8=A1=E5=9D=97?= =?UTF-8?q?=EF=BC=8C=E8=B0=83=E6=95=B4=E7=9B=AE=E5=BD=95=E7=BB=93=E6=9E=84?= =?UTF-8?q?=E4=B8=8E=E5=88=9D=E5=A7=8B=E5=8C=96=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 迁移存储模块功能到工作流模块，合并冗余代码 2. 调整环境配置加载路径与初始化脚本目录 3. 更新文档与前端组件代码适配重构 4. 新增工作流相关CRUD、模型与迁移文件 5. 移除过期模块与冗余代码，优化日志配置 --- .gitignore | 4 +- README.en.md | 2 +- README.md | 2 +- backend/README.md | 2 +- backend/alembic.ini | 35 +- backend/app/alembic/env.py | 81 +- backend/app/api/v1/core_backends.py | 100 + .../app/api/v1/module_ai/chat/controller.py | 160 +- backend/app/api/v1/module_ai/chat/service.py | 125 +- .../api/v1/module_common/health/controller.py | 106 +- .../api/v1/module_common/health/service.py | 88 + .../api/v1/module_generator/gencode/crud.py | 69 +- .../api/v1/module_generator/gencode/model.py | 17 +- .../v1/module_generator/gencode/service.py | 37 +- backend/app/api/v1/module_monitor/__init__.py | 2 - .../v1/module_monitor/resource/controller.py | 123 -- .../api/v1/module_monitor/resource/schema.py | 191 -- .../api/v1/module_monitor/resource/service.py | 471 ----- .../api/v1/module_monitor/server/service.py | 7 + backend/app/api/v1/module_storage/__init__.py | 11 - .../app/api/v1/module_storage/core/base.py | 94 - .../api/v1/module_storage/core/constants.py | 27 - .../api/v1/module_storage/core/cos_adapter.py | 141 -- .../app/api/v1/module_storage/core/encrypt.py | 35 - .../api/v1/module_storage/core/obs_adapter.py | 160 -- .../api/v1/module_storage/core/oss_adapter.py | 152 -- .../api/v1/module_storage/core/s3_adapter.py | 149 -- .../v1/module_storage/core/sftp_adapter.py | 143 -- .../api/v1/module_storage/file/controller.py | 103 - .../app/api/v1/module_storage/file/service.py | 203 -- backend/app/api/v1/module_storage/plugin.toml | 8 - .../api/v1/module_storage/transfer/engine.py | 263 --- .../api/v1/module_storage/transfer/service.py | 181 -- .../v1/module_storage/transfer/ws_manager.py | 34 - .../api/v1/module_system/auth/controller.py | 266 +-- .../v1/module_system/auth/oauth_service.py | 143 +- .../app/api/v1/module_system/auth/service.py | 264 ++- .../v1/module_system/auth/wx_mini_service.py | 130 +- .../api/v1/module_system/chat/controller.py | 53 +- .../app/api/v1/module_system/chat/schema.py | 9 + .../app/api/v1/module_system/chat/service.py | 145 +- .../api/v1/module_system/chat/ws_manager.py | 56 +- .../api/v1/module_system/log/controller.py | 4 +- .../app/api/v1/module_system/log/service.py | 4 +- .../api/v1/module_system/notice/service.py | 22 - .../v1/module_system/position/controller.py | 2 +- .../api/v1/module_system/position/service.py | 4 +- .../api/v1/module_system/role/controller.py | 2 +- backend/app/api/v1/module_system/role/crud.py | 57 +- .../app/api/v1/module_system/role/service.py | 49 +- .../api/v1/module_system/ticket/controller.py | 2 +- .../api/v1/module_system/ticket/service.py | 4 +- .../api/v1/module_system/user/controller.py | 2 +- backend/app/api/v1/module_system/user/crud.py | 18 +- .../app/api/v1/module_system/user/schema.py | 75 +- .../app/api/v1/module_system/user/service.py | 100 +- backend/app/api/v1/module_task/__init__.py | 15 +- .../v1/module_task/cronjob/job/controller.py | 37 +- .../api/v1/module_task/cronjob/job/schema.py | 31 + .../api/v1/module_task/cronjob/job/service.py | 97 +- .../v1/module_task/cronjob/node/controller.py | 6 +- .../api/v1/module_task/cronjob/node/schema.py | 8 + .../v1/module_task/cronjob/node/service.py | 24 +- .../workflow/core}/__init__.py | 0 .../api/v1/module_task/workflow/core/base.py | 667 +++++++ .../module_task/workflow/core/cos_adapter.py | 364 ++++ .../workflow}/core/factory.py | 17 +- .../workflow/core/ftp_ftps_adapter.py} | 126 +- .../workflow}/core/local_adapter.py | 62 +- .../module_task/workflow/core/obs_adapter.py | 487 +++++ .../module_task/workflow/core/oss_adapter.py | 285 +++ .../module_task/workflow/core/s3_adapter.py | 306 +++ .../module_task/workflow/core/sftp_adapter.py | 217 ++ .../workflow/flow}/__init__.py | 0 .../module_task/workflow/flow/controller.py | 94 + .../api/v1/module_task/workflow/flow/crud.py | 14 + .../api/v1/module_task/workflow/flow/model.py | 56 + .../v1/module_task/workflow/flow/schema.py | 178 ++ .../v1/module_task/workflow/flow/service.py | 486 +++++ .../v1/module_task/workflow/flows/__init__.py | 1 - .../module_task/workflow/flows/controller.py | 92 - .../api/v1/module_task/workflow/flows/crud.py | 90 - .../workflow/flows/handlers/__init__.py | 9 - .../workflow/flows/handlers/builtin_nodes.py | 348 ---- .../flows/handlers/workflow_engine.py | 136 -- .../v1/module_task/workflow/flows/model.py | 18 - .../v1/module_task/workflow/flows/schema.py | 119 -- .../v1/module_task/workflow/flows/service.py | 211 -- .../workflow/node}/__init__.py | 0 .../workflow/node}/controller.py | 44 +- .../workflow/node}/crud.py | 0 .../workflow/node}/model.py | 18 +- .../workflow/node}/schema.py | 67 +- .../workflow/node}/service.py | 92 +- .../workflow/node_type/__init__.py | 1 - .../workflow/node_type/controller.py | 97 - .../v1/module_task/workflow/node_type/crud.py | 102 - .../module_task/workflow/node_type/model.py | 22 - .../module_task/workflow/node_type/schema.py | 71 - .../module_task/workflow/node_type/service.py | 111 -- .../workflow/storage}/__init__.py | 0 .../workflow/storage/controller.py | 173 ++ .../v1/module_task/workflow/storage/schema.py | 23 + .../module_task/workflow/storage/service.py | 312 +++ .../workflow}/transfer/__init__.py | 0 .../workflow}/transfer/controller.py | 59 +- .../workflow}/transfer/crud.py | 0 .../module_task/workflow/transfer/engine.py | 377 ++++ .../workflow}/transfer/model.py | 9 +- .../workflow}/transfer/registry.py | 0 .../workflow}/transfer/schema.py | 47 + .../module_task/workflow/transfer/service.py | 255 +++ .../workflow/transfer/ws_manager.py | 5 + backend/app/config/path_conf.py | 2 +- backend/app/config/setting.py | 37 +- backend/app/core/ap_scheduler.py | 261 ++- backend/app/core/database.py | 15 +- backend/app/core/dependencies.py | 109 +- backend/app/core/discover.py | 144 +- backend/app/core/logger.py | 24 +- backend/app/core/permission.py | 82 +- backend/app/core/redis_crud.py | 29 + backend/app/core/router_class.py | 75 +- backend/app/core/validator.py | 37 + backend/app/core/ws_manager.py | 190 ++ backend/app/init_app.py | 83 +- .../plugin/module_example/demo/controller.py | 2 +- .../app/plugin/module_example/demo/service.py | 6 +- backend/app/scripts/initialize.py | 156 +- backend/app/utils/common_util.py | 21 +- backend/app/utils/crypto_util.py | 96 + backend/app/utils/excel_util.py | 19 + backend/app/utils/ip_local_util.py | 12 + backend/app/utils/password_util.py | 74 +- backend/env/.env.example | 2 +- backend/main.py | 1 + backend/pyproject.toml | 2 +- backend/requirements.txt | 4 +- backend/sql/{data => }/sys_dept.json | 0 backend/sql/{data => }/sys_dict_data.json | 0 backend/sql/{data => }/sys_dict_type.json | 0 backend/sql/{data => }/sys_menu.json | 885 +++------ backend/sql/{data => }/sys_param.json | 0 backend/sql/{data => }/sys_role.json | 0 backend/sql/{data => }/sys_user.json | 0 backend/sql/{data => }/sys_user_roles.json | 0 backend/sql/{data => }/sys_version.json | 0 backend/templates/vue/index.vue.jinja2 | 6 +- backend/tests/conftest.py | 1 + backend/uv.lock | 2 +- frontend/docs/src/en/guide/deployment.md | 2 +- frontend/docs/src/en/guide/start.md | 2 +- frontend/docs/src/guide/backend.md | 2 +- frontend/docs/src/guide/deployment.md | 2 +- frontend/docs/src/guide/start.md | 2 +- frontend/web/README.md | 194 +- .../web/src/api/module_monitor/resource.ts | 248 --- frontend/web/src/api/module_storage/file.ts | 71 - frontend/web/src/api/module_storage/source.ts | 111 -- frontend/web/src/api/module_system/chat.ts | 19 +- .../web/src/api/module_task/workflow/flow.ts | 116 +- .../web/src/api/module_task/workflow/node.ts | 164 ++ .../web/src/api/module_task/workflow/nodes.ts | 110 -- .../src/api/module_task/workflow/storage.ts | 155 ++ .../src/api/module_task/workflow/transfer.ts | 253 +++ .../components/cards/fa-card-grid/index.vue | 2 + .../components/display/fa-carousel/index.vue | 6 +- .../display/fa-result-page/index.vue | 5 +- .../components/display/fa-statistic/index.vue | 2 +- .../components/forms/fa-cascader/index.vue | 50 +- .../src/components/forms/fa-rate/index.vue | 40 +- .../components/forms/fa-time-select/index.vue | 52 +- .../components/forms/fa-transfer/index.vue | 66 - .../others/fa-infinite-scroll/index.vue | 5 +- .../components/others/fa-transfer/index.vue | 2 +- .../src/components/tables/fa-table/index.vue | 5 +- .../widgets/FaHorizontalSubmenu.vue | 5 +- .../layouts/fa-menus/fa-mixed-menu/index.vue | 5 +- .../widgets/FaSidebarSubmenu.vue | 11 +- .../web/src/layouts/fa-page-content/index.vue | 25 +- frontend/web/src/router/guards.ts | 5 +- frontend/web/src/router/routes.ts | 22 +- frontend/web/src/store/modules/chat.store.ts | 216 +- frontend/web/src/utils/download/index.ts | 21 - .../views/dashboard/home/modules/banner.vue | 64 +- .../dashboard/home/modules/image_cards.vue | 145 +- .../dashboard/home/modules/it_banners.vue | 131 +- .../dashboard/screen/modules/ScreenHeader.vue | 28 +- .../src/views/fastlink/current/profile.vue | 17 +- .../web/src/views/fastlink/fachat/index.vue | 263 +-- .../web/src/views/fastlink/tutorial/index.vue | 1396 ++++++------- .../views/fastlink/tutorial/manualSections.ts | 48 +- .../web/src/views/module_ai/chat/index.vue | 7 +- .../views/module_monitor/resource/index.vue | 652 ------ .../src/views/module_storage/file/index.vue | 353 ---- .../src/views/module_storage/source/index.vue | 607 ------ .../src/views/module_system/chat/index.vue | 201 +- .../src/views/module_system/menu/index.vue | 39 +- .../views/module_task/cronjob/job/index.vue | 28 +- .../flow/components/FaDynamicEdge.vue | 111 ++ .../flow/components/FaDynamicNode.vue | 270 +-- .../flow/components/FaEdgeConfigPanel.vue | 136 +- .../flow/components/FaFileBrowserDialog.vue | 198 ++ .../flow/components/FaNodeConfigPanel.vue | 282 --- .../flow/components/FaStorageNodePanel.vue | 194 ++ .../components/FaWorkflowDesignDrawer.vue | 1110 +++++------ .../workflow/flow/components/protocol.ts | 42 + .../views/module_task/workflow/flow/index.vue | 825 ++++++-- .../views/module_task/workflow/node/index.vue | 985 ++++++++++ .../module_task/workflow/nodes/index.vue | 683 ------- .../module_task/workflow/storage/index.vue | 1745 +++++++++++++++++ .../module_task/workflow/transfer/index.vue | 915 +++++++++ 212 files changed, 15329 insertions(+), 11742 deletions(-) create mode 100644 backend/app/api/v1/core_backends.py create mode 100644 backend/app/api/v1/module_common/health/service.py delete mode 100644 backend/app/api/v1/module_monitor/resource/controller.py delete mode 100644 backend/app/api/v1/module_monitor/resource/schema.py delete mode 100644 backend/app/api/v1/module_monitor/resource/service.py delete mode 100644 backend/app/api/v1/module_storage/__init__.py delete mode 100644 backend/app/api/v1/module_storage/core/base.py delete mode 100644 backend/app/api/v1/module_storage/core/constants.py delete mode 100644 backend/app/api/v1/module_storage/core/cos_adapter.py delete mode 100644 backend/app/api/v1/module_storage/core/encrypt.py delete mode 100644 backend/app/api/v1/module_storage/core/obs_adapter.py delete mode 100644 backend/app/api/v1/module_storage/core/oss_adapter.py delete mode 100644 backend/app/api/v1/module_storage/core/s3_adapter.py delete mode 100644 backend/app/api/v1/module_storage/core/sftp_adapter.py delete mode 100644 backend/app/api/v1/module_storage/file/controller.py delete mode 100644 backend/app/api/v1/module_storage/file/service.py delete mode 100644 backend/app/api/v1/module_storage/plugin.toml delete mode 100644 backend/app/api/v1/module_storage/transfer/engine.py delete mode 100644 backend/app/api/v1/module_storage/transfer/service.py delete mode 100644 backend/app/api/v1/module_storage/transfer/ws_manager.py rename backend/app/api/v1/{module_monitor/resource => module_task/workflow/core}/__init__.py (100%) create mode 100644 backend/app/api/v1/module_task/workflow/core/base.py create mode 100644 backend/app/api/v1/module_task/workflow/core/cos_adapter.py rename backend/app/api/v1/{module_storage => module_task/workflow}/core/factory.py (61%) rename backend/app/api/v1/{module_storage/core/ftp_adapter.py => module_task/workflow/core/ftp_ftps_adapter.py} (55%) rename backend/app/api/v1/{module_storage => module_task/workflow}/core/local_adapter.py (61%) create mode 100644 backend/app/api/v1/module_task/workflow/core/obs_adapter.py create mode 100644 backend/app/api/v1/module_task/workflow/core/oss_adapter.py create mode 100644 backend/app/api/v1/module_task/workflow/core/s3_adapter.py create mode 100644 backend/app/api/v1/module_task/workflow/core/sftp_adapter.py rename backend/app/api/v1/{module_storage/core => module_task/workflow/flow}/__init__.py (100%) create mode 100644 backend/app/api/v1/module_task/workflow/flow/controller.py create mode 100644 backend/app/api/v1/module_task/workflow/flow/crud.py create mode 100644 backend/app/api/v1/module_task/workflow/flow/model.py create mode 100644 backend/app/api/v1/module_task/workflow/flow/schema.py create mode 100644 backend/app/api/v1/module_task/workflow/flow/service.py delete mode 100644 backend/app/api/v1/module_task/workflow/flows/__init__.py delete mode 100644 backend/app/api/v1/module_task/workflow/flows/controller.py delete mode 100644 backend/app/api/v1/module_task/workflow/flows/crud.py delete mode 100644 backend/app/api/v1/module_task/workflow/flows/handlers/__init__.py delete mode 100644 backend/app/api/v1/module_task/workflow/flows/handlers/builtin_nodes.py delete mode 100644 backend/app/api/v1/module_task/workflow/flows/handlers/workflow_engine.py delete mode 100644 backend/app/api/v1/module_task/workflow/flows/model.py delete mode 100644 backend/app/api/v1/module_task/workflow/flows/schema.py delete mode 100644 backend/app/api/v1/module_task/workflow/flows/service.py rename backend/app/api/v1/{module_storage/file => module_task/workflow/node}/__init__.py (100%) rename backend/app/api/v1/{module_storage/source => module_task/workflow/node}/controller.py (76%) rename backend/app/api/v1/{module_storage/source => module_task/workflow/node}/crud.py (100%) rename backend/app/api/v1/{module_storage/source => module_task/workflow/node}/model.py (54%) rename backend/app/api/v1/{module_storage/source => module_task/workflow/node}/schema.py (55%) rename backend/app/api/v1/{module_storage/source => module_task/workflow/node}/service.py (72%) delete mode 100644 backend/app/api/v1/module_task/workflow/node_type/__init__.py delete mode 100644 backend/app/api/v1/module_task/workflow/node_type/controller.py delete mode 100644 backend/app/api/v1/module_task/workflow/node_type/crud.py delete mode 100644 backend/app/api/v1/module_task/workflow/node_type/model.py delete mode 100644 backend/app/api/v1/module_task/workflow/node_type/schema.py delete mode 100644 backend/app/api/v1/module_task/workflow/node_type/service.py rename backend/app/api/v1/{module_storage/source => module_task/workflow/storage}/__init__.py (100%) create mode 100644 backend/app/api/v1/module_task/workflow/storage/controller.py create mode 100644 backend/app/api/v1/module_task/workflow/storage/schema.py create mode 100644 backend/app/api/v1/module_task/workflow/storage/service.py rename backend/app/api/v1/{module_storage => module_task/workflow}/transfer/__init__.py (100%) rename backend/app/api/v1/{module_storage => module_task/workflow}/transfer/controller.py (73%) rename backend/app/api/v1/{module_storage => module_task/workflow}/transfer/crud.py (100%) create mode 100644 backend/app/api/v1/module_task/workflow/transfer/engine.py rename backend/app/api/v1/{module_storage => module_task/workflow}/transfer/model.py (85%) rename backend/app/api/v1/{module_storage => module_task/workflow}/transfer/registry.py (100%) rename backend/app/api/v1/{module_storage => module_task/workflow}/transfer/schema.py (61%) create mode 100644 backend/app/api/v1/module_task/workflow/transfer/service.py create mode 100644 backend/app/api/v1/module_task/workflow/transfer/ws_manager.py create mode 100644 backend/app/core/ws_manager.py create mode 100644 backend/app/utils/crypto_util.py rename backend/sql/{data => }/sys_dept.json (100%) rename backend/sql/{data => }/sys_dict_data.json (100%) rename backend/sql/{data => }/sys_dict_type.json (100%) rename backend/sql/{data => }/sys_menu.json (87%) rename backend/sql/{data => }/sys_param.json (100%) rename backend/sql/{data => }/sys_role.json (100%) rename backend/sql/{data => }/sys_user.json (100%) rename backend/sql/{data => }/sys_user_roles.json (100%) rename backend/sql/{data => }/sys_version.json (100%) delete mode 100644 frontend/web/src/api/module_monitor/resource.ts delete mode 100644 frontend/web/src/api/module_storage/file.ts delete mode 100644 frontend/web/src/api/module_storage/source.ts create mode 100644 frontend/web/src/api/module_task/workflow/node.ts delete mode 100644 frontend/web/src/api/module_task/workflow/nodes.ts create mode 100644 frontend/web/src/api/module_task/workflow/storage.ts create mode 100644 frontend/web/src/api/module_task/workflow/transfer.ts delete mode 100644 frontend/web/src/components/forms/fa-transfer/index.vue delete mode 100644 frontend/web/src/views/module_monitor/resource/index.vue delete mode 100644 frontend/web/src/views/module_storage/file/index.vue delete mode 100644 frontend/web/src/views/module_storage/source/index.vue create mode 100644 frontend/web/src/views/module_task/workflow/flow/components/FaDynamicEdge.vue create mode 100644 frontend/web/src/views/module_task/workflow/flow/components/FaFileBrowserDialog.vue delete mode 100644 frontend/web/src/views/module_task/workflow/flow/components/FaNodeConfigPanel.vue create mode 100644 frontend/web/src/views/module_task/workflow/flow/components/FaStorageNodePanel.vue create mode 100644 frontend/web/src/views/module_task/workflow/flow/components/protocol.ts create mode 100644 frontend/web/src/views/module_task/workflow/node/index.vue delete mode 100644 frontend/web/src/views/module_task/workflow/nodes/index.vue create mode 100644 frontend/web/src/views/module_task/workflow/storage/index.vue create mode 100644 frontend/web/src/views/module_task/workflow/transfer/index.vue diff --git a/.gitignore b/.gitignore index dcdacacf..b22dbf4a 100755 --- a/.gitignore +++ b/.gitignore @@ -45,11 +45,11 @@ node_modules backend/logs backend/*.db -backend/alembic/versions/* +# 用户上传目录，禁止入库 +backend/static/upload/ backend/env/.env.dev backend/env/.env.prod backend/env/.env.test -!backend/alembic/versions/.gitkeep docker/.env docker/nginx/app/dist diff --git a/README.en.md b/README.en.md index dccd6a44..15709d22 100644 --- a/README.en.md +++ b/README.en.md @@ -57,7 +57,7 @@ English | [简体中文](./README.md) git clone https://github.com/fastapiadmin/FastapiAdmin.git # 2. Configure environments -cp backend/env/.env.dev.example backend/env/.env.dev +cp backend/env/.env.example backend/env/.env.dev cp frontend/web/.env.development.example frontend/web/.env.development # 3. Start backend (auto-creates tables + seed data on first run) diff --git a/README.md b/README.md index dd7e04ff..a4a79800 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ git clone https://gitee.com/fastapiadmin/FastapiAdmin.git # 2. 配置环境 -cp backend/env/.env.dev.example backend/env/.env.dev +cp backend/env/.env.example backend/env/.env.dev cp frontend/web/.env.development.example frontend/web/.env.development # 3. 启动后端（首次自动建表 + 初始化数据） diff --git a/backend/README.md b/backend/README.md index 48278233..3ffc56b3 100644 --- a/backend/README.md +++ b/backend/README.md @@ -72,7 +72,7 @@ module_*/ ### 第一次在本机跑起来 -1. 复制 `env/.env.dev.example` → `env/.env.dev`，填写数据库、Redis 等（先在 DB 中建好空库）。 +1. 复制 `env/.env.example` → `env/.env.dev`，填写数据库、Redis 等（先在 DB 中建好空库）。 2. 在 **`backend/` 目录下** 安装依赖：推荐 **`uv sync`**；或 `pip install -r requirements.txt`。 3. **启动**：`uv run main.py run --env=dev`（或 `python main.py run --env=dev`）。**首次启动会自动初始化数据库表与基础数据**，一般**无需**先执行 `upgrade`。接口文档示例：`http://127.0.0.1:8001/docs`（端口见 `.env.dev` 中 `SERVER_PORT`）。 diff --git a/backend/alembic.ini b/backend/alembic.ini index 63b22c7f..bea2ff3c 100644 --- a/backend/alembic.ini +++ b/backend/alembic.ini @@ -83,37 +83,4 @@ sqlalchemy.url = driver://user:pass@localhost/dbname # ruff.executable = %(here)s/.venv/bin/ruff # ruff.options = --fix REVISION_SCRIPT_FILENAME -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARNING -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARNING -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S +# 日志配置已移除：alembic 日志统一由 app.core.logger（loguru）接管，不走 ini 的 fileConfig。 diff --git a/backend/app/alembic/env.py b/backend/app/alembic/env.py index 620d0c75..a69b793b 100644 --- a/backend/app/alembic/env.py +++ b/backend/app/alembic/env.py @@ -1,9 +1,11 @@ import asyncio -from logging.config import fileConfig +import warnings from alembic import context -from sqlalchemy import pool +from alembic.runtime.migration import MigrationContext +from sqlalchemy import pool, text from sqlalchemy.engine import Connection +from sqlalchemy.exc import SAWarning from sqlalchemy.ext.asyncio import create_async_engine from app.config.path_conf import ALEMBIC_VERSION_DIR @@ -11,58 +13,26 @@ from app.config.setting import settings from app.core.base_model import MappedBase from app.utils.import_util import ImportUtil -# 确保 alembic 版本目录存在 ALEMBIC_VERSION_DIR.mkdir(parents=True, exist_ok=True) -# 清除MappedBase.metadata中的表定义，避免重复注册 -if hasattr(MappedBase, "metadata") and MappedBase.metadata.tables: - print(f"🧹 清除已存在的表定义，当前有 {len(MappedBase.metadata.tables)} 个表") - # 创建一个新的空metadata对象 - from sqlalchemy import MetaData - - MappedBase.metadata = MetaData() - print("✅️ 已重置metadata") - -# 自动查找所有模型 print("🔍 开始查找模型...") found_models = ImportUtil.find_models(MappedBase) print(f"📊 找到 {len(found_models)} 个有效模型") -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. alembic_config = context.config -# Interpret the config file for Python logging. -# This line sets up loggers basically. -if alembic_config.config_file_name is not None: - fileConfig(alembic_config.config_file_name) +warnings.filterwarnings( + "ignore", + message=r"Cannot correctly sort tables.*", + category=SAWarning, +) -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata target_metadata = MappedBase.metadata - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. alembic_config.set_main_option("sqlalchemy.url", settings.ASYNC_DB_URI) def run_migrations_offline() -> None: - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - - Calls to context.execute() here emit the given string to the - script output. - - 返回: - - None + """离线模式运行迁移 """ url = alembic_config.get_main_option("sqlalchemy.url") # 确保URL不为None @@ -81,13 +51,7 @@ def run_migrations_offline() -> None: def run_migrations_online() -> None: - """Run migrations in 'online' mode. - - In this scenario we need to create an Engine - and associate a connection with the context. - - 返回: - - None + """异步模式运行迁移 """ url = alembic_config.get_main_option("sqlalchemy.url") # 确保URL不为None @@ -102,7 +66,13 @@ def run_migrations_online() -> None: await connectable.dispose() def do_run_migrations(connection: Connection) -> None: - def process_revision_directives(context, revision, directives) -> None: + # MySQL 建表时外键引用的表必须已存在；sys_dept/sys_user 循环外键（互相引用）无法顺序建表， + # PG/SQLite 支持 forward reference 无此限制。执行迁移时对 MySQL 临时关闭外键检查， + # 该变量为会话级，连接关闭后自动恢复，不影响运行时。 + if connection.dialect.name == "mysql": + connection.execute(text("SET FOREIGN_KEY_CHECKS=0")) + + def process_revision_directives(context: MigrationContext, revision: str, directives: list) -> None: script = directives[0] # 检查所有操作集是否为空 @@ -115,17 +85,28 @@ def run_migrations_online() -> None: else: print("✅️ 检测到模型变更，生成迁移文件") + def include_name(name, type_, parent_names) -> bool: + # 只对 MappedBase 中存在的表做 autogenerate 对比，自动忽略数据库中的非模型表 + # （apscheduler_jobs、alembic_version 及未来新增的任何非模型表）， + # 避免被误判为多余表而生成 DROP。官方推荐范式（include_name 过滤表名）， + # 优于逐个硬编码排除。 + if type_ == "table": + return name in target_metadata.tables + return True + context.configure( connection=connection, target_metadata=target_metadata, compare_type=True, compare_server_default=True, transaction_per_migration=True, + include_name=include_name, process_revision_directives=process_revision_directives, ) - with context.begin_transaction(): - context.run_migrations() + + context.run_migrations() + connection.commit() asyncio.run(run_async_migrations()) diff --git a/backend/app/api/v1/core_backends.py b/backend/app/api/v1/core_backends.py new file mode 100644 index 00000000..6f7c4d12 --- /dev/null +++ b/backend/app/api/v1/core_backends.py @@ -0,0 +1,100 @@ +"""api → core 能力登记（应用组合根唯一注入点）。 + +依赖方向约定：core 层保持纯基础层，不得反向 ``import app.api.*``； +本模块（api 层）由组合根 ``init_app.register_routers`` 调用 ``register_core_backends()``， +把 api 层的实现登记回 core 的注入点（操作日志落库、用户加载、数据权限、调度器）。 +core 至多通过 set_* 持有回调，永不持有业务模型引用。 +""" + +from typing import Any + +from app.core.logger import logger + + +async def _operation_log_writer(log_data: dict[str, Any]) -> None: + """操作日志落库实现：仅持久化，异常由 core 调用方兜底记录。""" + from app.api.v1.module_system.log.crud import OperationLogCRUD + from app.api.v1.module_system.log.schema import OperationLogCreateSchema + from app.core.base_schema import AuthSchema + from app.core.database import async_db_session + + async with async_db_session() as session, session.begin(): + await OperationLogCRUD(AuthSchema(), session).create(data=OperationLogCreateSchema(**log_data)) + + +async def _load_user_row(db: Any, user_id: int) -> Any | None: + """按 ID 加载未删除用户行（HTTP / WebSocket 认证时校验用户仍存在）。""" + from sqlalchemy import select + + from app.api.v1.module_system.user.model import UserModel + + result = await db.execute(select(UserModel).where(UserModel.id == user_id, UserModel.is_deleted == False)) # noqa: E712 + return result.scalars().first() + + +async def _load_user_role_scopes(db: Any, user_id: int) -> set[int]: + """读取用户全部角色的数据权限范围集合（data_scope 值）。""" + from sqlalchemy import select + + from app.api.v1.module_system.role.model import RoleModel + from app.api.v1.module_system.user.model import UserModel + + stmt = select(RoleModel.data_scope).join(RoleModel.users).where(UserModel.id == user_id) + rows = (await db.execute(stmt)).scalars().all() + return {int(scope) for scope in rows} + + +async def _load_dept_children(db: Any, dept_id: int) -> set[int]: + """按部门树计算某部门的子部门 ID 集合（含自身，语义与历史实现一致）。""" + from sqlalchemy import select + + from app.api.v1.module_system.dept.model import DeptModel + from app.utils.common_util import get_child_id_map, get_child_recursion + + dept_objs = (await db.execute(select(DeptModel))).scalars().all() + return get_child_recursion(id=dept_id, id_map=get_child_id_map(dept_objs)) + + +def _record_job_failure(record: dict[str, Any]) -> None: + """任务执行失败落库（APScheduler 事件线程同步执行，仅持久化）。""" + from sqlalchemy.orm import Session + + from app.api.v1.module_task.cronjob.job.model import JobModel + from app.core.database import engine + + with Session(engine) as session: + job_log = JobModel(**record) + session.add(job_log) + session.commit() + logger.info(f"失败日志已记录: job_id={record['job_id']}, id={job_log.id}") + + +def _register_system_jobs() -> None: + """登记系统级周期任务（调度器取得持有权后由 core 回调，幂等）。""" + from apscheduler.triggers.cron import CronTrigger + + from app.api.v1.module_system.log.service import OperationLogService + from app.core.ap_scheduler import SchedulerUtil + + SchedulerUtil.register_system_job( + "system_cleanup_operation_log", + OperationLogService.cleanup_operation_log, + trigger=CronTrigger(day_of_week="sun", hour=3, minute=0), + name="操作日志清理", + ) + + +def register_core_backends() -> None: + """应用组合根调用：把 api 层能力登记进 core 注入点。""" + from app.core.ap_scheduler import set_scheduler_backends + from app.core.dependencies import set_user_loader + from app.core.permission import set_data_scope_loaders + from app.core.router_class import set_operation_log_writer + + set_operation_log_writer(_operation_log_writer) + set_user_loader(_load_user_row) + set_data_scope_loaders(_load_user_role_scopes, _load_dept_children) + set_scheduler_backends( + system_job_registrar=_register_system_jobs, + job_failure_recorder=_record_job_failure, + ) diff --git a/backend/app/api/v1/module_ai/chat/controller.py b/backend/app/api/v1/module_ai/chat/controller.py index cc560025..636bf147 100644 --- a/backend/app/api/v1/module_ai/chat/controller.py +++ b/backend/app/api/v1/module_ai/chat/controller.py @@ -8,8 +8,7 @@ from redis.asyncio import Redis from app.common.response import ResponseSchema, SuccessResponse from app.core.base_schema import AuthSchema, PaginationQueryParam -from app.core.database import async_db_session -from app.core.dependencies import AuthPermission, _authenticate, redis_getter +from app.core.dependencies import AuthPermission, redis_getter, websocket_authenticate from app.core.exceptions import CustomException from app.core.logger import logger from app.core.router_class import OperationLogRoute @@ -114,7 +113,7 @@ async def list_model_config_controller( auth: Annotated[AuthSchema, Security(AuthPermission(["module_ai:chat:query"]))], ) -> JSONResponse: service = AiModelConfigService(auth, redis) - result = await service.list() + result = await service.list_configs() return SuccessResponse(data=result, msg="获取模型配置列表成功") @@ -186,37 +185,21 @@ async def websocket_chat_controller(websocket: WebSocket) -> None: - 对话：{"message": "...", "session_id": "...", "files": [...]} - 停止：{"action": "stop", "session_id": "..."} - ws://127.0.0.1:8001/api/v1/ai/chat/ws?token=xxx - """ - # 接收客户端 subprotocol：约定客户端在 Sec-WebSocket-Protocol 中以 "access_token." 携带 - # 推荐方式：subprotocol 不会进 URL，不出现在 Nginx access log / 浏览器历史 / 抓包日志 - # 同时兼容旧版：用 query_params 传 token（不推荐，仅作向后兼容） - # - # 浏览器侧示例： - # new WebSocket(url, ["access_token", "access_token." + jwt]) - # Python websocket-client 示例： - # websockets.connect(url, subprotocols=["access_token", f"access_token.{jwt}"]) - token = None - use_subprotocol = False - if websocket.headers.get("sec-websocket-protocol"): - for proto in websocket.headers["sec-websocket-protocol"].split(","): - proto = proto.strip() - if proto.startswith("access_token."): - token = proto[len("access_token.") :] - use_subprotocol = True - break - if not token: - # 旧版/非浏览器客户端兼容：保留 query ?token= - token = websocket.query_params.get("token") + ws://127.0.0.1:8001/api/v1/ai/chat/ws - if not token: - await _send_error_and_close(websocket, "未提供认证token，请重新登录") + 令牌优先通过 Sec-WebSocket-Protocol 携带（不会进入网关/服务的 access log）： + new WebSocket(url, ["access_token", "access_token." + jwt]) + 小程序等无法自定义子协议的客户端，可用 ?token= 兜底。 + """ + # 握手阶段完成认证；未 accept 前无法发送业务报文，失败直接关闭 + try: + auth, subprotocol = await websocket_authenticate(websocket) + except Exception as e: + logger.warning("WebSocket认证失败: {}", e) + await websocket.close(code=4001, reason="无效令牌") return - if use_subprotocol: - await websocket.accept(subprotocol="access_token") - else: - await websocket.accept() + await websocket.accept(subprotocol=subprotocol) # 跨消息循环共享的停止信号：客户端发送 stop 时 set，生成器检测到后退出 stop_event = asyncio.Event() @@ -225,76 +208,73 @@ async def websocket_chat_controller(websocket: WebSocket) -> None: try: redis = websocket.app.state.redis - async with async_db_session() as db: - auth = await _authenticate(token, db, redis) + logger.info("WebSocket连接已建立: {} - 用户: {}", websocket.client, auth.user.username or "未认证") - logger.info("WebSocket连接已建立: {} - 用户: {}", websocket.client, auth.user.username or "未认证") + chat_service = ChatService(auth) - chat_service = ChatService(auth) - - # 消息循环 - while True: + # 消息循环 + while True: + try: + data = await websocket.receive_text() try: - data = await websocket.receive_text() - try: - message_data = json.loads(data) - query = ChatQuerySchema(**message_data) - except json.JSONDecodeError: - logger.warning("收到非JSON消息: {}", data) - await websocket.send_text("消息格式错误，请发送JSON格式的消息") - continue - except Exception as e: - logger.warning("消息校验失败: {}", e) - await websocket.send_text(f"消息格式错误: {e}") - continue + message_data = json.loads(data) + query = ChatQuerySchema(**message_data) + except json.JSONDecodeError: + logger.warning("收到非JSON消息: {}", data) + await websocket.send_text("消息格式错误，请发送JSON格式的消息") + continue + except Exception as e: + logger.warning("消息校验失败: {}", e) + await websocket.send_text(f"消息格式错误: {e}") + continue - # 处理停止指令 - if query.action == "stop": - if is_generating.is_set(): - stop_event.set() - logger.info("收到停止指令: session={}", query.session_id) - await websocket.send_text("[STOPPED]") - else: - await websocket.send_text("当前没有正在进行的生成任务") - continue + # 处理停止指令 + if query.action == "stop": + if is_generating.is_set(): + stop_event.set() + logger.info("收到停止指令: session={}", query.session_id) + await websocket.send_text("[STOPPED]") + else: + await websocket.send_text("当前没有正在进行的生成任务") + continue - # 对话指令 - logger.info("收到聊天查询: session_id={}", query.session_id) + # 对话指令 + logger.info("收到聊天查询: session_id={}", query.session_id) - is_generating.set() + is_generating.set() + stop_event.clear() + # 读取用户的 AI 模型配置（每次可动态切换） + model_config = await get_user_model_config(redis, auth.user.id) + try: + async for chunk in chat_service.chat_query( + query=query, + stop_event=stop_event, + model_config=model_config, + ): + if not chunk: + continue + try: + await websocket.send_text(chunk) + except RuntimeError: + logger.warning("WebSocket连接已关闭，停止发送消息") + return + finally: + is_generating.clear() stop_event.clear() - # 读取用户的 AI 模型配置（每次可动态切换） - model_config = await get_user_model_config(redis, auth.user.id) - try: - async for chunk in chat_service.chat_query( - query=query, - stop_event=stop_event, - model_config=model_config, - ): - if not chunk: - continue - try: - await websocket.send_text(chunk) - except RuntimeError: - logger.warning("WebSocket连接已关闭，停止发送消息") - return - finally: - is_generating.clear() - stop_event.clear() - # 告知前端生成结束 - try: - await websocket.send_text("[DONE]") - except RuntimeError: - return - - except WebSocketDisconnect: - logger.info("WebSocket连接已断开: {}", websocket.client) + # 告知前端生成结束 + try: + await websocket.send_text("[DONE]") + except RuntimeError: return + except WebSocketDisconnect: + logger.info("WebSocket连接已断开: {}", websocket.client) + return + except CustomException as e: - # 认证失败等业务异常 - logger.warning("WebSocket认证失败: {}", e.msg) + # 业务异常（认证已前置，多为会话/模型配置异常） + logger.warning("WebSocket业务异常: {}", e.msg) await _send_error_and_close(websocket, e.msg) except Exception as e: # 未知异常 diff --git a/backend/app/api/v1/module_ai/chat/service.py b/backend/app/api/v1/module_ai/chat/service.py index ee0c551c..a4b188ad 100644 --- a/backend/app/api/v1/module_ai/chat/service.py +++ b/backend/app/api/v1/module_ai/chat/service.py @@ -1,6 +1,7 @@ import asyncio import json from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager from datetime import datetime from typing import Any @@ -17,6 +18,7 @@ from app.core.exceptions import CustomException from app.core.logger import logger from app.core.redis_crud import RedisCURD from app.utils.ai_factory import AgnoFactory +from app.utils.crypto_util import CryptoUtil from .crud import ChatSessionCRUD from .schema import ( @@ -237,7 +239,7 @@ class ChatService: if json_end > json_start: json_str = response_text[json_start:json_end].strip() action = json.loads(json_str) - except (json.JSONDecodeError, Exception): + except Exception: pass if not action: @@ -348,6 +350,21 @@ class ChatService: _AI_MODEL_TTL = 604800 # AI 模型配置缓存 7 天，不活跃用户自动清理 +_AI_MODEL_LOCK_TTL = 10 # 读改写锁最长持有时间(秒) + + +@asynccontextmanager +async def _model_config_lock(redis: Redis, user_id: int) -> AsyncGenerator[None, None]: + """用户模型配置读改写的分布式锁：配置以 JSON list 整体存取，并发写会互相覆盖。""" + crud = RedisCURD(redis) + key = f"{RedisInitKeyConfig.AI_MODEL_CONFIG.key}:lock:{user_id}" + acquired, token = await crud.lock(key=key, expire=_AI_MODEL_LOCK_TTL) + if not acquired: + raise CustomException(msg="模型配置正在被修改，请稍后重试") + try: + yield + finally: + await crud.unlock(key=key, value=token) def _ai_model_items_key(user_id: int) -> str: @@ -358,6 +375,26 @@ def _ai_model_active_key(user_id: int) -> str: return f"{RedisInitKeyConfig.AI_MODEL_CONFIG.key}:active:{user_id}" +# 配置项中需要静态加密的敏感字段：Redis 中的 api_key 一律以密文存储 +_SECRET_FIELD = "api_key" + + +def _seal_item(item: dict[str, Any]) -> dict[str, Any]: + """落盘前加密 api_key，其余字段保持明文（用于展示与检索）。""" + sealed = dict(item) + if sealed.get(_SECRET_FIELD): + sealed[_SECRET_FIELD] = CryptoUtil.encrypt(sealed[_SECRET_FIELD]) + return sealed + + +def _open_item(item: dict[str, Any]) -> dict[str, Any]: + """读取后解密 api_key；加密能力上线前的历史明文原样返回。""" + opened = dict(item) + if opened.get(_SECRET_FIELD): + opened[_SECRET_FIELD] = CryptoUtil.decrypt_or_keep(opened[_SECRET_FIELD]) + return opened + + async def get_user_model_config(redis: Redis, user_id: int) -> dict[str, Any] | None: """读取当前激活的 AI 模型配置；不存在或未激活返回 None。""" active_id = await RedisCURD(redis).get(_ai_model_active_key(user_id)) @@ -370,8 +407,8 @@ async def get_user_model_config(redis: Redis, user_id: int) -> dict[str, Any] | return None -async def list_user_model_configs(redis: Redis, user_id: int) -> list[dict[str, Any]]: - """列出用户的所有模型配置项。""" +async def _read_raw_items(redis: Redis, user_id: int) -> list[dict[str, Any]]: + """读取存储层的原始配置项（api_key 为密文）。""" raw = await RedisCURD(redis).get(_ai_model_items_key(user_id)) if not raw: return [] @@ -385,6 +422,20 @@ async def list_user_model_configs(redis: Redis, user_id: int) -> list[dict[str, return [] +async def _write_raw_items(redis: Redis, user_id: int, items: list[dict[str, Any]]) -> None: + """写入配置项：加密敏感字段后整体落盘（调用方需持有读改写锁）。""" + await RedisCURD(redis).set( + _ai_model_items_key(user_id), + json.dumps([_seal_item(it) for it in items], ensure_ascii=False), + expire=_AI_MODEL_TTL, + ) + + +async def list_user_model_configs(redis: Redis, user_id: int) -> list[dict[str, Any]]: + """列出用户的所有模型配置项（api_key 已解密为明文）。""" + return [_open_item(it) for it in await _read_raw_items(redis, user_id)] + + async def get_active_model_id(redis: Redis, user_id: int) -> str | None: """读取当前激活的模型配置 ID；为空表示使用系统默认。""" return await RedisCURD(redis).get(_ai_model_active_key(user_id)) @@ -397,24 +448,20 @@ async def create_user_model_config( ) -> dict[str, Any]: """新增一个模型配置项。""" import uuid - from datetime import datetime - items = await list_user_model_configs(redis, user_id) - item = { - **config.model_dump(), - "id": uuid.uuid4().hex, - "created_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), - } - items.append(item) - await RedisCURD(redis).set( - _ai_model_items_key(user_id), - json.dumps(items, ensure_ascii=False), - expire=_AI_MODEL_TTL, - ) + async with _model_config_lock(redis, user_id): + items = await list_user_model_configs(redis, user_id) + item = { + **config.model_dump(), + "id": uuid.uuid4().hex, + "created_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + } + items.append(item) + await _write_raw_items(redis, user_id, items) - # 若用户尚未激活任何配置，自动激活新增的 - if not await get_active_model_id(redis, user_id): - await RedisCURD(redis).set(_ai_model_active_key(user_id), item["id"], expire=_AI_MODEL_TTL) + # 若用户尚未激活任何配置，自动激活新增的 + if not await get_active_model_id(redis, user_id): + await RedisCURD(redis).set(_ai_model_active_key(user_id), item["id"], expire=_AI_MODEL_TTL) logger.info("已新增 AI 模型配置: user_id={} name={} id={}", user_id, config.name, item["id"]) return item @@ -427,34 +474,28 @@ async def update_user_model_config( config: AiModelConfigSchema, ) -> dict[str, Any] | None: """更新指定 ID 的模型配置项；不存在返回 None。""" - items = await list_user_model_configs(redis, user_id) - target = next((it for it in items if it.get("id") == config_id), None) - if not target: - return None - target.update(config.model_dump()) - await RedisCURD(redis).set( - _ai_model_items_key(user_id), - json.dumps(items, ensure_ascii=False), - expire=_AI_MODEL_TTL, - ) + async with _model_config_lock(redis, user_id): + items = await list_user_model_configs(redis, user_id) + target = next((it for it in items if it.get("id") == config_id), None) + if not target: + return None + target.update(config.model_dump()) + await _write_raw_items(redis, user_id, items) logger.info("已更新 AI 模型配置: user_id={} id={}", user_id, config_id) return target async def delete_user_model_config(redis: Redis, user_id: int, config_id: str) -> bool: """删除指定 ID 的模型配置项；若该 ID 是当前激活则清空激活。""" - items = await list_user_model_configs(redis, user_id) - new_items = [it for it in items if it.get("id") != config_id] - if len(new_items) == len(items): - return False - await RedisCURD(redis).set( - _ai_model_items_key(user_id), - json.dumps(new_items, ensure_ascii=False), - expire=_AI_MODEL_TTL, - ) - active_id = await get_active_model_id(redis, user_id) - if active_id == config_id: - await RedisCURD(redis).delete(_ai_model_active_key(user_id)) + async with _model_config_lock(redis, user_id): + items = await list_user_model_configs(redis, user_id) + new_items = [it for it in items if it.get("id") != config_id] + if len(new_items) == len(items): + return False + await _write_raw_items(redis, user_id, new_items) + active_id = await get_active_model_id(redis, user_id) + if active_id == config_id: + await RedisCURD(redis).delete(_ai_model_active_key(user_id)) logger.info("已删除 AI 模型配置: user_id={} id={}", user_id, config_id) return True @@ -484,7 +525,7 @@ class AiModelConfigService: def _user_id(self) -> int: return self.auth.user.id - async def list(self) -> dict[str, Any]: + async def list_configs(self) -> dict[str, Any]: """获取配置列表 + 当前激活 ID。""" items = await list_user_model_configs(self.redis, self._user_id) active_id = await get_active_model_id(self.redis, self._user_id) diff --git a/backend/app/api/v1/module_common/health/controller.py b/backend/app/api/v1/module_common/health/controller.py index e7b852fe..b2dff80f 100644 --- a/backend/app/api/v1/module_common/health/controller.py +++ b/backend/app/api/v1/module_common/health/controller.py @@ -1,70 +1,21 @@ import asyncio -import shutil -import time from collections.abc import AsyncIterable from datetime import datetime from fastapi import APIRouter, Request from fastapi.responses import JSONResponse from fastapi.sse import EventSourceResponse, ServerSentEvent -from sqlalchemy import text from app.common.enums import RET from app.common.response import ErrorResponse, ResponseSchema, SuccessResponse from app.config.setting import settings -from app.core.database import async_db_session -from app.core.logger import logger from app.core.router_class import OperationLogRoute -from .schema import DependencyStatus, HealthOut, ReadinessOut +from .schema import HealthOut, ReadinessOut +from .service import HEALTH_STREAM_INTERVAL, HealthService HealthRouter = APIRouter(route_class=OperationLogRoute, prefix="/health", tags=["健康检查"]) -# ── 健康检查时间间隔 ── -_HEALTH_STREAM_INTERVAL = 30 # 秒 - - -async def _check_database() -> DependencyStatus: - """检查数据库连接""" - try: - start = time.perf_counter() - async with async_db_session() as session: - await session.execute(text("SELECT 1")) - latency = (time.perf_counter() - start) * 1000 - return DependencyStatus(status=1, latency_ms=round(latency, 2)) - except Exception as e: - logger.warning(f"数据库健康检查失败: {e}") - return DependencyStatus(status=0) - - -async def _check_redis(request: Request) -> DependencyStatus: - """检查 Redis 连接""" - try: - redis = getattr(request.app.state, "redis", None) - if not redis: - return DependencyStatus(status=0) - - start = time.perf_counter() - await redis.ping() - latency = (time.perf_counter() - start) * 1000 - return DependencyStatus(status=1, latency_ms=round(latency, 2)) - except Exception as e: - logger.warning(f"Redis 健康检查失败: {e}") - return DependencyStatus(status=0) - - -def _get_disk_usage() -> float: - """获取磁盘使用率""" - try: - usage = shutil.disk_usage("/") - return round(usage.used / usage.total * 100, 1) - except Exception: - return -1.0 - - -# 应用启动时间戳 -_start_time = datetime.now() - @HealthRouter.get("/check", summary="健康检查", response_model=ResponseSchema[HealthOut]) async def health_check() -> JSONResponse: @@ -76,13 +27,12 @@ async def health_check() -> JSONResponse: 返回: - SuccessResponse: 包含进程存活状态、启动时间、版本号的 JSON 响应。 """ - uptime = (datetime.now() - _start_time).total_seconds() return SuccessResponse( data=HealthOut( status=1, timestamp=datetime.now().isoformat(), version=settings.VERSION, - uptime_seconds=uptime, + uptime_seconds=HealthService.get_uptime(), ), msg="系统健康", ) @@ -98,13 +48,12 @@ async def liveness_check() -> JSONResponse: 返回: - SuccessResponse: 包含进程存活状态、启动时间、版本号的 JSON 响应。 """ - uptime = (datetime.now() - _start_time).total_seconds() return SuccessResponse( data=HealthOut( status=1, timestamp=datetime.now().isoformat(), version=settings.VERSION, - uptime_seconds=uptime, + uptime_seconds=HealthService.get_uptime(), ), msg="进程存活", ) @@ -120,31 +69,19 @@ async def readiness_check(request: Request) -> JSONResponse: 返回: - SuccessResponse | ErrorResponse: 依赖就绪时返回 200，未就绪返回 503。 """ - uptime = (datetime.now() - _start_time).total_seconds() - - db_status, redis_status = await asyncio.gather( - _check_database(), - _check_redis(request), - ) - - dependencies = { - "database": db_status, - "redis": redis_status, - } + redis = getattr(request.app.state, "redis", None) + dependencies = await HealthService.check_dependencies(redis) # 判断总体状态 - def is_ok(d: DependencyStatus) -> bool: - return d.status == 1 - - all_ok = all(is_ok(d) for d in dependencies.values()) + all_ok = all(d.status == 1 for d in dependencies.values()) payload = ReadinessOut( status=1 if all_ok else 0, timestamp=datetime.now().isoformat(), version=settings.VERSION, - uptime_seconds=uptime, + uptime_seconds=HealthService.get_uptime(), dependencies=dependencies, - disk_usage=_get_disk_usage(), + disk_usage=HealthService.get_disk_usage(), ) if all_ok: @@ -164,29 +101,12 @@ async def readiness_check(request: Request) -> JSONResponse: # ============================================================ -async def _build_health_payload(request: Request) -> dict: - """采集当前健康状态""" - db_status, redis_status = await asyncio.gather( - _check_database(), - _check_redis(request), - ) - return { - "status": 1 if db_status.status and redis_status.status else 0, - "dependencies": { - "database": db_status.model_dump(), - "redis": redis_status.model_dump(), - }, - "disk_usage": _get_disk_usage(), - "uptime_seconds": (datetime.now() - _start_time).total_seconds(), - "timestamp": datetime.now().isoformat(), - } - - @HealthRouter.get("/stream", summary="健康状态实时推送", response_class=EventSourceResponse) async def health_stream(request: Request) -> AsyncIterable[ServerSentEvent]: """SSE 实时推送健康状态，每 30 秒推送一次，客户端无需轮询 /ready。""" - yield ServerSentEvent(data=await _build_health_payload(request), event="health") + redis = getattr(request.app.state, "redis", None) + yield ServerSentEvent(data=await HealthService.collect_status(redis), event="health") while True: - await asyncio.sleep(_HEALTH_STREAM_INTERVAL) - yield ServerSentEvent(data=await _build_health_payload(request), event="health") + await asyncio.sleep(HEALTH_STREAM_INTERVAL) + yield ServerSentEvent(data=await HealthService.collect_status(redis), event="health") diff --git a/backend/app/api/v1/module_common/health/service.py b/backend/app/api/v1/module_common/health/service.py new file mode 100644 index 00000000..9e703884 --- /dev/null +++ b/backend/app/api/v1/module_common/health/service.py @@ -0,0 +1,88 @@ +import asyncio +import shutil +import time +from datetime import datetime +from typing import Any + +from sqlalchemy import text + +from app.core.database import async_db_session +from app.core.logger import logger + +from .schema import DependencyStatus + +# 应用启动时间戳 +_start_time = datetime.now() + +# 健康检查时间间隔 +HEALTH_STREAM_INTERVAL = 30 # 秒 + + +class HealthService: + """健康检查服务：封装数据库 / Redis / 磁盘等依赖的探活逻辑。""" + + @staticmethod + async def check_database() -> DependencyStatus: + """检查数据库连接""" + try: + start = time.perf_counter() + async with async_db_session() as session: + await session.execute(text("SELECT 1")) + latency = (time.perf_counter() - start) * 1000 + return DependencyStatus(status=1, latency_ms=round(latency, 2)) + except Exception as e: + logger.warning(f"数据库健康检查失败: {e}") + return DependencyStatus(status=0) + + @staticmethod + async def check_redis(redis: Any | None) -> DependencyStatus: + """检查 Redis 连接 + + 参数: + - redis: Redis 客户端实例（由控制器从应用状态中取出注入）。 + """ + if redis is None: + return DependencyStatus(status=0) + try: + start = time.perf_counter() + await redis.ping() + latency = (time.perf_counter() - start) * 1000 + return DependencyStatus(status=1, latency_ms=round(latency, 2)) + except Exception as e: + logger.warning(f"Redis 健康检查失败: {e}") + return DependencyStatus(status=0) + + @staticmethod + async def check_dependencies(redis: Any | None) -> dict[str, DependencyStatus]: + """并行探活全部依赖项。""" + db_status, redis_status = await asyncio.gather( + HealthService.check_database(), + HealthService.check_redis(redis), + ) + return {"database": db_status, "redis": redis_status} + + @staticmethod + def get_disk_usage() -> float: + """获取磁盘使用率""" + try: + usage = shutil.disk_usage("/") + return round(usage.used / usage.total * 100, 1) + except Exception: + return -1.0 + + @staticmethod + def get_uptime() -> float: + """进程运行时长（秒）""" + return (datetime.now() - _start_time).total_seconds() + + @staticmethod + async def collect_status(redis: Any | None) -> dict: + """采集当前健康状态（供 SSE 实时推送）。""" + dependencies = await HealthService.check_dependencies(redis) + return { + "status": 1 if all(d.status == 1 for d in dependencies.values()) else 0, + "dependencies": {name: dep.model_dump() for name, dep in dependencies.items()}, + "disk_usage": HealthService.get_disk_usage(), + "uptime_seconds": HealthService.get_uptime(), + "timestamp": datetime.now().isoformat(), + } diff --git a/backend/app/api/v1/module_generator/gencode/crud.py b/backend/app/api/v1/module_generator/gencode/crud.py index df357faa..acbce134 100644 --- a/backend/app/api/v1/module_generator/gencode/crud.py +++ b/backend/app/api/v1/module_generator/gencode/crud.py @@ -15,7 +15,6 @@ from app.utils.common_util import search_to_dict from .model import GenTableColumnModel, GenTableModel from .schema import ( GenDBTableSchema, - GenTableColumnOutSchema, GenTableColumnSchema, GenTableQueryParam, GenTableSchema, @@ -130,11 +129,12 @@ class GenTableCRUD(CRUDBase[GenTableModel, GenTableSchema, GenTableSchema]): """ await self.delete(ids=ids) - async def get_db_table_list(self, search: GenTableQueryParam | None = None) -> list[dict]: + async def get_db_table_list(self, table_name: str | None = None, table_comment: str | None = None) -> list[dict]: """根据查询参数获取数据库表列表信息。 参数: - - search (GenTableQueryParam | None): 查询参数对象。 + - table_name (str | None): 表名关键字（模糊匹配）。 + - table_comment (str | None): 表注释关键字（模糊匹配）。 返回: - list[dict]: 数据库表列表信息（已转为可序列化字典）。 @@ -145,6 +145,9 @@ class GenTableCRUD(CRUDBase[GenTableModel, GenTableSchema, GenTableSchema]): inspector: Inspector = inspect(engine) table_names = inspector.get_table_names() + name_kw = table_name.strip() if table_name else "" + comment_kw = table_comment.strip() if table_comment else "" + dict_data = [] for table_name in table_names: try: @@ -155,14 +158,12 @@ class GenTableCRUD(CRUDBase[GenTableModel, GenTableSchema, GenTableSchema]): logger.warning(f"获取表 {table_name} 的注释失败: {e}") table_comment = "" - # 统一处理 search 为 None 的情况，避免重复判断 - if search: - # 表名过滤：忽略大小写，支持模糊匹配 - if search.table_name and search.table_name[1] and search.table_name[1].lower() not in table_name.lower(): - continue - # 表注释过滤：忽略大小写，支持模糊匹配；table_comment 为 None 时视为空字符串 - if search.table_comment and search.table_comment[1] and search.table_comment[1] not in table_comment: - continue + # 表名过滤：忽略大小写，支持模糊匹配 + if name_kw and name_kw.lower() not in table_name.lower(): + continue + # 表注释过滤：忽略大小写，支持模糊匹配；table_comment 为 None 时视为空字符串 + if comment_kw and comment_kw not in table_comment: + continue table_info = { "database_name": database_name, @@ -177,9 +178,10 @@ class GenTableCRUD(CRUDBase[GenTableModel, GenTableSchema, GenTableSchema]): async def get_db_table_page( self, - search: GenTableQueryParam | None, - offset: int, - limit: int, + table_name: str | None = None, + table_comment: str | None = None, + offset: int = 0, + limit: int = 10, ) -> tuple[list[dict], int]: """数据库侧分页获取物理表列表（用于导入表弹窗）。 @@ -189,7 +191,8 @@ class GenTableCRUD(CRUDBase[GenTableModel, GenTableSchema, GenTableSchema]): - 若方言不支持，则回退到旧的全量遍历。 参数: - - search (GenTableQueryParam | None): 表名/注释过滤条件。 + - table_name (str | None): 表名关键字（模糊匹配）。 + - table_comment (str | None): 表注释关键字（模糊匹配）。 - offset (int): 偏移量。 - limit (int): 每页条数。 @@ -199,19 +202,8 @@ class GenTableCRUD(CRUDBase[GenTableModel, GenTableSchema, GenTableSchema]): database_name = settings.DATABASE_NAME db_type = (settings.DATABASE_TYPE or "").lower() - # 解析 like 关键字（GenTableQueryParam 把字段包装成 ("like", value)） - name_kw = None - comment_kw = None - if search: - try: - if search.table_name and search.table_name[1]: - name_kw = str(search.table_name[1]).strip() - if search.table_comment and search.table_comment[1]: - comment_kw = str(search.table_comment[1]).strip() - except Exception: - # 兜底：参数结构异常时忽略过滤 - name_kw = None - comment_kw = None + name_kw = table_name.strip() if table_name else None + comment_kw = table_comment.strip() if table_comment else None # MySQL / MariaDB if db_type in {"mysql", "mariadb"}: @@ -277,7 +269,7 @@ class GenTableCRUD(CRUDBase[GenTableModel, GenTableSchema, GenTableSchema]): return items, total # Fallback：回退旧逻辑（全量遍历再分页由上层处理） - all_items = await self.get_db_table_list(search) + all_items = await self.get_db_table_list(table_name=name_kw, table_comment=comment_kw) total = len(all_items) return all_items[offset : offset + limit], total @@ -493,14 +485,17 @@ class GenTableColumnCRUD(CRUDBase[GenTableColumnModel, GenTableColumnSchema, Gen """ return await self.get_list(search={"table_id": table_id}, order_by=order_by, preload=preload) - async def get_gen_db_table_columns_by_name(self, table_name: str | None) -> list[GenTableColumnOutSchema]: - """根据业务表名称获取业务表字段列表信息。 + async def get_gen_db_table_columns_by_name(self, table_name: str | None) -> list[dict]: + """获取物理数据库表的列信息（原始字典）。 + + 说明: + - 仅返回 DB 元信息的原生字典，模型转换（OutSchema）由 Service 层完成。 参数: - - table_name (str | None): 业务表名称。 + - table_name (str | None): 物理表名称。 返回: - - list[GenTableColumnOutSchema]: 业务表字段列表信息对象。 + - list[dict]: 列信息字典列表。 """ # 检查表名是否为空 if not table_name: @@ -508,19 +503,13 @@ class GenTableColumnCRUD(CRUDBase[GenTableColumnModel, GenTableColumnSchema, Gen try: # 在线程池中执行同步 inspect 操作，避免阻塞事件循环 - columns_info = await asyncio.to_thread( + return await asyncio.to_thread( GenTableColumnCRUD._sync_get_table_columns, settings.DATABASE_TYPE, table_name, ) - - # 转换为GenTableColumnOutSchema对象列表 - columns_list = [GenTableColumnOutSchema(**column_info) for column_info in columns_info] - - return columns_list except Exception as e: logger.error(f"获取表{table_name}的字段列表时出错: {e!s}") - # 确保即使出错也返回空列表而不是None raise async def list_gen_table_column_crud( diff --git a/backend/app/api/v1/module_generator/gencode/model.py b/backend/app/api/v1/module_generator/gencode/model.py index 7336b57f..da4f11b4 100644 --- a/backend/app/api/v1/module_generator/gencode/model.py +++ b/backend/app/api/v1/module_generator/gencode/model.py @@ -1,6 +1,5 @@ from sqlalchemy import Boolean, ForeignKey, Integer, String, Text from sqlalchemy.orm import Mapped, mapped_column, relationship, validates -from sqlalchemy.sql import expression from app.config.setting import settings from app.core.base_model import ModelMixin, UserMixin @@ -54,16 +53,16 @@ class GenTableColumnModel(ModelMixin, UserMixin): column_type: Mapped[str] = mapped_column(String(100), nullable=False, comment="列类型") column_length: Mapped[str | None] = mapped_column(String(50), nullable=True, comment="列长度") column_default: Mapped[str | None] = mapped_column(String(200), nullable=True, comment="列默认值") - is_pk: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=expression.false(), comment="是否主键") - is_increment: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=expression.false(), comment="是否自增") - is_nullable: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=expression.true(), comment="是否允许为空") - is_unique: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=expression.false(), comment="是否唯一") + is_pk: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=SqlalchemyUtil.get_boolean_server_default(settings.DATABASE_TYPE, False), comment="是否主键") + is_increment: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=SqlalchemyUtil.get_boolean_server_default(settings.DATABASE_TYPE, False), comment="是否自增") + is_nullable: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=SqlalchemyUtil.get_boolean_server_default(settings.DATABASE_TYPE, True), comment="是否允许为空") + is_unique: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=SqlalchemyUtil.get_boolean_server_default(settings.DATABASE_TYPE, False), comment="是否唯一") python_type: Mapped[str | None] = mapped_column(String(100), nullable=True, comment="Python类型") python_field: Mapped[str | None] = mapped_column(String(200), nullable=True, comment="Python字段名") - is_insert: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=expression.true(), comment="是否为新增字段") - is_edit: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=expression.true(), comment="是否编辑字段") - is_list: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=expression.true(), comment="是否列表字段") - is_query: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=expression.false(), comment="是否查询字段") + is_insert: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=SqlalchemyUtil.get_boolean_server_default(settings.DATABASE_TYPE, True), comment="是否为新增字段") + is_edit: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=SqlalchemyUtil.get_boolean_server_default(settings.DATABASE_TYPE, True), comment="是否编辑字段") + is_list: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=SqlalchemyUtil.get_boolean_server_default(settings.DATABASE_TYPE, True), comment="是否列表字段") + is_query: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=SqlalchemyUtil.get_boolean_server_default(settings.DATABASE_TYPE, False), comment="是否查询字段") query_type: Mapped[str | None] = mapped_column(String(50), nullable=True, default=None, comment="查询方式") html_type: Mapped[str | None] = mapped_column(String(100), nullable=True, default="input", comment="前端显示类型") dict_type: Mapped[str | None] = mapped_column(String(200), nullable=True, default="", comment="前端对应字典类型") diff --git a/backend/app/api/v1/module_generator/gencode/service.py b/backend/app/api/v1/module_generator/gencode/service.py index ef485a7a..646be70c 100644 --- a/backend/app/api/v1/module_generator/gencode/service.py +++ b/backend/app/api/v1/module_generator/gencode/service.py @@ -232,7 +232,7 @@ class GenTableService: return gen_table @handle_service_exception - async def get_gen_table_list(self, search: GenTableQueryParam) -> list[dict]: + async def get_gen_table_list(self, search: GenTableQueryParam) -> list[GenTableOutSchema]: """获取代码生成业务表列表信息。 参数: @@ -240,10 +240,10 @@ class GenTableService: - search (GenTableQueryParam): 查询参数模型。 返回: - - list[dict]: 包含业务表列表信息的字典列表。 + - list[GenTableOutSchema]: 包含业务表列表信息的模型列表。 """ gen_table_list_result = await GenTableCRUD(self.auth, self.db).get_gen_table_list(search, preload=["columns"]) - return [GenTableOutSchema.model_validate(obj).model_dump() for obj in gen_table_list_result] + return [GenTableOutSchema.model_validate(obj) for obj in gen_table_list_result] @handle_service_exception async def get_gen_table_page( @@ -287,7 +287,7 @@ class GenTableService: 返回: - list[Any]: 包含数据库表列表信息的任意类型列表。 """ - gen_db_table_list_result = await GenTableCRUD(self.auth, self.db).get_db_table_list(search) + gen_db_table_list_result = await GenTableCRUD(self.auth, self.db).get_db_table_list(table_name=search.table_name, table_comment=search.table_comment) return gen_db_table_list_result @handle_service_exception @@ -309,7 +309,12 @@ class GenTableService: - dict[str, Any]: 含 items、total、has_next 等字段。 """ offset = (page_no - 1) * page_size - items, total = await GenTableCRUD(self.auth, self.db).get_db_table_page(search=search, offset=offset, limit=page_size) + items, total = await GenTableCRUD(self.auth, self.db).get_db_table_page( + table_name=search.table_name, + table_comment=search.table_comment, + offset=offset, + limit=page_size, + ) return { "items": items, "total": total, @@ -336,6 +341,11 @@ class GenTableService: return result + async def _get_db_table_columns(self, table_name: str) -> list[GenTableColumnOutSchema]: + """读取物理库表字段并转换为输出模型（供导入/同步/预览使用）。""" + columns_info = await GenTableColumnCRUD(self.auth, self.db).get_gen_db_table_columns_by_name(table_name) + return [GenTableColumnOutSchema(**column_info) for column_info in columns_info] + @handle_service_exception async def import_gen_table(self, gen_table_list: list[GenTableOutSchema]) -> bool: """导入表结构到生成器。 @@ -363,7 +373,7 @@ class GenTableService: if not table.columns: table.columns = [] add_gen_table = await GenTableCRUD(self.auth, self.db).add_gen_table(GenTableSchema.model_validate(table.model_dump())) - gen_table_columns = await GenTableColumnCRUD(self.auth, self.db).get_gen_db_table_columns_by_name(table_name) + gen_table_columns = await self._get_db_table_columns(table_name) if len(gen_table_columns) > 0: table.id = add_gen_table.id for column in gen_table_columns: @@ -938,11 +948,10 @@ class GenTableService: table_columns = table.columns or [] table_column_map = {column.column_name: column for column in table_columns} # 确保db_table_columns始终是列表类型，避免None值 - db_table_columns = await GenTableColumnCRUD(self.auth, self.db).get_gen_db_table_columns_by_name(table_name) or [] + db_table_columns = await self._get_db_table_columns(table_name) db_table_columns = [col for col in db_table_columns if col is not None] db_table_column_names = [column.column_name for column in db_table_columns] try: - # 参考 RuoYi：同步 DB 元信息，但尽量保留用户“生成配置”字段（dict/html/query/python_field...） preserve_keys = { "dict_type", "query_type", @@ -1065,9 +1074,9 @@ class GenTableService: # 2) 回退：仅从 DB 读取结构（只读，无法配置子表字段） try: - gen_table_columns = await GenTableColumnCRUD(self.auth, self.db).get_gen_db_table_columns_by_name(sub_name_raw) + gen_table_columns = await self._get_db_table_columns(sub_name_raw) except Exception as e: - logger.warning(f"获取子表 {sub_name_raw} 字段失败: {e!s}") + gen_table_columns = await GenTableColumnCRUD(self.auth, self.db).get_gen_db_table_columns_by_name(sub_name_raw) gen_table.sub = False gen_table.sub_table = None gen_table.master_sub_hint = f"无法读取子表结构：{e!s}" @@ -1188,9 +1197,9 @@ class GenTableService: if not table.id: raise CustomException(msg="业务表ID不能为空") - db_cols = await GenTableColumnCRUD(self.auth, self.db).get_gen_db_table_columns_by_name(table_name) + db_cols = await self._get_db_table_columns(table_name) added, removed, changed, unchanged = self._sync_preview_diff( - current_cols=table.columns or [], + db_cols = await GenTableColumnCRUD(self.auth, self.db).get_gen_db_table_columns_by_name(table_name) db_cols=db_cols or [], ) preview = GenSyncPreviewSchema( @@ -1212,9 +1221,9 @@ class GenTableService: cur_sub_cols = GenTableOutSchema.model_validate(sub_cfg).columns or [] else: cur_sub_cols = [] - db_sub_cols = await GenTableColumnCRUD(self.auth, self.db).get_gen_db_table_columns_by_name(sn) + db_sub_cols = await self._get_db_table_columns(sn) s_added, s_removed, s_changed, s_unchanged = self._sync_preview_diff( - current_cols=cur_sub_cols, + db_sub_cols = await GenTableColumnCRUD(self.auth, self.db).get_gen_db_table_columns_by_name(sn) db_cols=db_sub_cols or [], ) preview.sub = GenSyncPreviewSchema( diff --git a/backend/app/api/v1/module_monitor/__init__.py b/backend/app/api/v1/module_monitor/__init__.py index 3113ec57..0dddba0c 100644 --- a/backend/app/api/v1/module_monitor/__init__.py +++ b/backend/app/api/v1/module_monitor/__init__.py @@ -2,12 +2,10 @@ from fastapi import APIRouter from .cache.controller import CacheRouter from .online.controller import OnlineRouter -from .resource.controller import ResourceRouter from .server.controller import ServerRouter monitor_router = APIRouter(prefix="/monitor") monitor_router.include_router(CacheRouter) monitor_router.include_router(OnlineRouter) -monitor_router.include_router(ResourceRouter) monitor_router.include_router(ServerRouter) diff --git a/backend/app/api/v1/module_monitor/resource/controller.py b/backend/app/api/v1/module_monitor/resource/controller.py deleted file mode 100644 index fb768f01..00000000 --- a/backend/app/api/v1/module_monitor/resource/controller.py +++ /dev/null @@ -1,123 +0,0 @@ -from typing import Annotated - -from fastapi import APIRouter, Body, Depends, File, Form, Query, Request, Security, UploadFile, status -from fastapi.responses import FileResponse, JSONResponse, StreamingResponse - -from app.api.v1.module_common.file.service import FileService -from app.common.request import PaginationService -from app.common.response import ResponseSchema, StreamResponse, SuccessResponse, UploadFileResponse -from app.core.base_schema import PaginationQueryParam, UploadResponseSchema -from app.core.dependencies import AuthPermission -from app.core.router_class import OperationLogRoute -from app.utils.common_util import bytes2file_response - -from .schema import ResourceCopySchema, ResourceCreateDirSchema, ResourceItemSchema, ResourceMoveSchema, ResourceRenameSchema, ResourceSearchQueryParam -from .service import ResourceService - -ResourceRouter = APIRouter(route_class=OperationLogRoute, prefix="/resource", tags=["资源管理"]) - - -@ResourceRouter.get("/list", summary="获取目录列表", response_model=ResponseSchema[list[ResourceItemSchema]], dependencies=[Security(AuthPermission(["module_monitor:resource:query"]))]) -async def get_directory_list_controller( - request: Request, - page: Annotated[PaginationQueryParam, Depends()], - search: Annotated[ResourceSearchQueryParam, Query()], -) -> JSONResponse: - result_dict_list = await ResourceService.get_resources_list(search=search, base_url=str(request.base_url)) - result_dict = await PaginationService.paginate( - data_list=result_dict_list, - page_no=page.page_no, - page_size=page.page_size, - ) - return SuccessResponse(data=result_dict, msg="获取目录列表成功") - - -@ResourceRouter.post("/upload", summary="上传文件", response_model=ResponseSchema[UploadResponseSchema], dependencies=[Security(AuthPermission(["module_monitor:resource:upload"]))]) -async def upload_file_controller( - request: Request, - file: Annotated[UploadFile, File(description="上传文件")], - target_path: Annotated[str | None, Form(description="目标目录路径")] = None, -) -> JSONResponse: - result = await FileService.upload_service( - base_url=str(request.base_url), - file=file, - upload_type="resource", - target_path=target_path, - ) - return SuccessResponse(data=result, msg="上传文件成功") - - -@ResourceRouter.get( - "/download", - summary="下载文件", - dependencies=[Security(AuthPermission(["module_monitor:resource:download"]))], -) -async def download_file_controller( - path: Annotated[str, Query(description="文件路径")], -) -> FileResponse: - file_path = await ResourceService.download_file(file_path=path) - - import os - - filename = os.path.basename(file_path) - - return UploadFileResponse( - file_path=file_path, - filename=filename, - media_type="application/octet-stream", - ) - - -@ResourceRouter.delete("/delete", summary="删除文件", response_model=ResponseSchema[None], dependencies=[Security(AuthPermission(["module_monitor:resource:delete"]))]) -async def delete_files_controller( - paths: Annotated[list[str], Body(description="文件路径列表")], -) -> JSONResponse: - await ResourceService.delete_file(paths=paths) - return SuccessResponse(msg="删除文件成功") - - -@ResourceRouter.post("/move", summary="移动文件", response_model=ResponseSchema[None], dependencies=[Security(AuthPermission(["module_monitor:resource:move"]))]) -async def move_file_controller( - data: Annotated[ResourceMoveSchema, Body(description="移动文件参数")], -) -> JSONResponse: - await ResourceService.move_file(data=data) - return SuccessResponse(msg="移动文件成功") - - -@ResourceRouter.post("/copy", summary="复制文件", response_model=ResponseSchema[None], dependencies=[Security(AuthPermission(["module_monitor:resource:copy"]))]) -async def copy_file_controller( - data: Annotated[ResourceCopySchema, Body(description="复制文件参数")], -) -> JSONResponse: - await ResourceService.copy_file(data=data) - return SuccessResponse(msg="复制文件成功") - - -@ResourceRouter.post("/rename", summary="重命名文件", response_model=ResponseSchema[None], dependencies=[Security(AuthPermission(["module_monitor:resource:rename"]))]) -async def rename_file_controller( - data: Annotated[ResourceRenameSchema, Body(description="重命名文件参数")], -) -> JSONResponse: - await ResourceService.rename_file(data=data) - return SuccessResponse(msg="重命名文件成功") - - -@ResourceRouter.post("/mkdir", status_code=status.HTTP_201_CREATED, summary="创建目录", response_model=ResponseSchema[None], dependencies=[Security(AuthPermission(["module_monitor:resource:mkdir"]))]) -async def create_directory_controller( - data: Annotated[ResourceCreateDirSchema, Body(description="创建目录参数")], -) -> JSONResponse: - await ResourceService.create_directory(data=data) - return SuccessResponse(msg="创建目录成功") - - -@ResourceRouter.post("/export", summary="导出资源列表", dependencies=[Security(AuthPermission(["module_monitor:resource:export"]))]) -async def export_resource_list_controller( - request: Request, - search: Annotated[ResourceSearchQueryParam, Query()], -) -> StreamingResponse: - result_dict_list = await ResourceService.get_resources_list(search=search, base_url=str(request.base_url)) - export_result = await ResourceService.export_resource(data_list=result_dict_list) - - return StreamResponse( - data=bytes2file_response(export_result), - media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - headers={"Content-Disposition": "attachment; filename=resource_list.xlsx"}, - ) diff --git a/backend/app/api/v1/module_monitor/resource/schema.py b/backend/app/api/v1/module_monitor/resource/schema.py deleted file mode 100644 index f95e40e7..00000000 --- a/backend/app/api/v1/module_monitor/resource/schema.py +++ /dev/null @@ -1,191 +0,0 @@ -from datetime import datetime -from urllib.parse import urlparse - -from pydantic import ( - BaseModel, - ConfigDict, - Field, - field_validator, - model_validator, -) - - -class ResourceItemSchema(BaseModel): - """资源项目模型""" - - model_config = ConfigDict(from_attributes=True) - - name: str = Field(..., description="文件名") - file_url: str = Field(..., description="文件URL路径") - relative_path: str = Field(..., description="相对路径") - is_file: bool = Field(..., description="是否为文件") - is_dir: bool = Field(..., description="是否为目录") - size: int | None = Field(None, description="文件大小(字节)") - created_time: datetime | None = Field(None, description="创建时间") - modified_time: datetime | None = Field(None, description="修改时间") - is_hidden: bool = Field(False, description="是否为隐藏文件") - - @field_validator("file_url") - @classmethod - def _validate_file_url(cls, v: str) -> str: - v = v.strip() - parsed = urlparse(v) - # 允许相对路径（以 / 开头）和完整的 http/https URL - if parsed.scheme and parsed.scheme not in ("http", "https"): - raise ValueError("文件URL必须为 http/https 或相对路径") - return v - - @field_validator("relative_path") - @classmethod - def _validate_relative_path(cls, v: str) -> str: - v = v.strip() - if ".." in v or v.startswith("\\"): - raise ValueError("相对路径包含不安全字符") - return v - - @model_validator(mode="after") - def _validate_flags(self): - if self.is_file and self.is_dir: - raise ValueError("不能同时为文件和目录") - if not self.is_file and not self.is_dir: - raise ValueError("必须是文件或目录之一") - # 根据名称自动修正隐藏标记 - self.is_hidden = self.name.startswith(".") - return self - - -class ResourceDirectorySchema(BaseModel): - """资源目录模型""" - - model_config = ConfigDict(from_attributes=True) - - path: str = Field(..., description="目录路径") - name: str = Field(..., description="目录名称") - items: list[ResourceItemSchema] = Field(default_factory=list, description="目录项") - total_files: int = Field(0, description="文件总数") - total_dirs: int = Field(0, description="目录总数") - total_size: int = Field(0, description="总大小") - - -class ResourceUploadSchema(BaseModel): - """资源上传响应模型""" - - model_config = ConfigDict(from_attributes=True) - - filename: str = Field(..., description="文件名") - file_url: str = Field(..., description="访问URL") - file_size: int = Field(..., description="文件大小") - upload_time: datetime = Field(..., description="上传时间") - - -class ResourceMoveSchema(BaseModel): - """资源移动模型""" - - model_config = ConfigDict(from_attributes=True) - - source_path: str = Field(..., description="源路径") - target_path: str = Field(..., description="目标路径") - overwrite: bool = Field(False, description="是否覆盖") - - @field_validator("source_path", "target_path") - @classmethod - def validate_paths(cls, value: str): - """校验移动/复制涉及的源路径与目标路径非空并去首尾空格。 - - 参数: - - value (str): 路径字段当前值。 - - 返回: - - str: 去空格后的路径。 - - 异常: - - ValueError: 路径为空时抛出。 - """ - if not value or len(value.strip()) == 0: - raise ValueError("路径不能为空") - return value.strip() - - -class ResourceCopySchema(ResourceMoveSchema): - """资源复制模型""" - - -class ResourceRenameSchema(BaseModel): - """资源重命名模型""" - - model_config = ConfigDict(from_attributes=True) - - old_path: str = Field(..., description="原路径") - new_name: str = Field(..., max_length=255, description="新名称") - - @field_validator("old_path", "new_name") - @classmethod - def validate_inputs(cls, value: str): - """校验重命名所需的原路径与新名称非空并去首尾空格。 - - 参数: - - value (str): 字段当前值。 - - 返回: - - str: 去空格后的值。 - - 异常: - - ValueError: 值为空时抛出。 - """ - if not value or len(value.strip()) == 0: - raise ValueError("参数不能为空") - return value.strip() - - @field_validator("new_name") - @classmethod - def _validate_new_name(cls, v: str) -> str: - v = v.strip() - if ".." in v or "/" in v or "\\" in v: - raise ValueError("新名称包含不安全字符") - return v - - -class ResourceCreateDirSchema(BaseModel): - """创建目录模型""" - - model_config = ConfigDict(from_attributes=True) - - parent_path: str = Field(..., description="父目录路径") - dir_name: str = Field(..., description="目录名称", max_length=255) - - @field_validator("parent_path", "dir_name") - @classmethod - def validate_inputs(cls, value: str, info): - """校验创建目录的父路径与目录名，防止路径遍历等不安全输入。 - - 参数: - - value (str): 当前字段值。 - - info: Pydantic 校验上下文（含 `field_name`）。 - - 返回: - - str: 规范化后的字段值。 - - 异常: - - ValueError: 含不安全字符或目录名为空时抛出。 - """ - # 对于parent_path允许为空字符串（表示根目录）或 '/'，其他情况必须非空 - if info.field_name == "parent_path": - # 对于parent_path仍然严格检查路径遍历 - if ".." in value or value.startswith("\\"): - raise ValueError("参数包含不安全字符") - else: # 对于dir_name仍然严格检查 - if not value or len(value.strip()) == 0: - raise ValueError("参数不能为空") - if ".." in value or value.startswith(("/", "\\")): - raise ValueError("参数包含不安全字符") - return value.strip() - - -class ResourceSearchQueryParam(BaseModel): - """资源搜索查询参数""" - - name: str | None = Field(None, description="搜索关键词") - path: str | None = Field(None, description="目录路径") - include_hidden: bool = Field(False, description="是否包含隐藏文件") - - diff --git a/backend/app/api/v1/module_monitor/resource/service.py b/backend/app/api/v1/module_monitor/resource/service.py deleted file mode 100644 index 1e43db3d..00000000 --- a/backend/app/api/v1/module_monitor/resource/service.py +++ /dev/null @@ -1,471 +0,0 @@ -import ast -import os -import re -import shutil -import urllib.parse -from datetime import datetime -from pathlib import Path -from urllib.parse import urlparse - -from app.config.path_conf import STATIC_DIR -from app.config.setting import settings -from app.core.exceptions import CustomException -from app.core.logger import logger -from app.utils.excel_util import ExcelUtil - -from .schema import ( - ResourceCopySchema, - ResourceCreateDirSchema, - ResourceItemSchema, - ResourceMoveSchema, - ResourceRenameSchema, - ResourceSearchQueryParam, -) - - -class ResourceService: - """资源管理模块服务层 - 管理系统静态文件目录（仅管理 upload 目录）""" - - MAX_UPLOAD_SIZE = 100 * 1024 * 1024 # 100MB - MAX_SEARCH_RESULTS = 1000 - MAX_PATH_DEPTH = 20 - - @staticmethod - def _get_resource_root() -> str: - resource_root = os.path.join(str(STATIC_DIR), "upload") - os.makedirs(resource_root, exist_ok=True) - return resource_root - - @staticmethod - def _get_safe_path(path: str | None = None) -> str: - resource_root = ResourceService._get_resource_root() - - if not path or not isinstance(path, str): - return resource_root - - static_prefix = settings.STATIC_URL.rstrip("/") - root_prefix = settings.ROOT_PATH.rstrip("/") if getattr(settings, "ROOT_PATH", "") else "" - root_static_prefix = f"{root_prefix}{static_prefix}" if root_prefix else static_prefix - - def strip_prefix(p: str) -> str: - if p.startswith(root_static_prefix): - return p[len(root_static_prefix) :].lstrip("/") - if p.startswith(static_prefix): - return p[len(static_prefix) :].lstrip("/") - return p - - if path.startswith(("http://", "https://")): - parsed = urlparse(path) - url_path = parsed.path or "" - path = strip_prefix(url_path) - else: - path = strip_prefix(path) - - path = path.strip().replace("//", "/").replace("\\\\\\\\", "/").replace("\\\\", "/") - - path = path.removeprefix("/") - - path = path.removeprefix("upload/") - - if ".." in path or "\x00" in path: - logger.error(f"检测到路径遍历攻击尝试: {path}") - raise CustomException(msg="非法的路径格式") - - decoded_path = urllib.parse.unquote(path) - if ".." in decoded_path: - logger.error(f"检测到编码后的路径遍历攻击: {path}") - raise CustomException(msg="非法的路径格式") - - safe_path = os.path.normpath(os.path.join(resource_root, path)) - - resource_root_abs = os.path.normpath(os.path.abspath(resource_root)) - safe_path_abs = os.path.normpath(os.path.abspath(safe_path)) - - if not safe_path_abs.startswith(resource_root_abs + os.sep) and safe_path_abs != resource_root_abs: - logger.error(f"路径遍历攻击被阻止: 尝试访问 {safe_path_abs}, 但根目录是 {resource_root_abs}") - raise CustomException(msg="访问路径不在允许范围内") - - try: - relative_path = os.path.relpath(safe_path_abs, resource_root_abs) - if relative_path.count(os.sep) > ResourceService.MAX_PATH_DEPTH: - raise CustomException(msg="路径深度超过限制") - except ValueError: - raise CustomException(msg="无效的路径") - - return safe_path_abs - - @staticmethod - def _path_exists(path: str) -> bool: - try: - safe_path = ResourceService._get_safe_path(path) - return os.path.exists(safe_path) - except Exception as e: - raise CustomException(msg=f"检查路径是否存在失败: {e!s}") - - @staticmethod - def _sanitize_filename(filename: str) -> str: - if not filename: - return f"file_{datetime.now().strftime('%Y%m%d%H%M%S')}" - - dangerous_patterns = [ - r"\.\.", - r"[\/]", - r"\x00", - r"%2e%2e", - r"%252e%252e", - ] - for pattern in dangerous_patterns: - if re.search(pattern, filename, re.IGNORECASE): - logger.error(f"检测到文件名路径遍历攻击: {filename}") - return f"file_{datetime.now().strftime('%Y%m%d%H%M%S')}" - - decoded = urllib.parse.unquote(filename) - decoded_twice = urllib.parse.unquote(decoded) - for check in [decoded, decoded_twice]: - if ".." in check or "/" in check or "\\" in check: - logger.error(f"检测到编码后的文件名攻击: {filename}") - return f"file_{datetime.now().strftime('%Y%m%d%H%M%S')}" - - filename = os.path.basename(filename) - filename = re.sub(r'[<>:"|?*\x00-\x1f]', "", filename) - filename = re.sub(r"\.{2,}", ".", filename) - filename = filename.strip(". ") - - if not filename: - filename = f"file_{datetime.now().strftime('%Y%m%d%H%M%S')}" - - return filename - - @staticmethod - def _detect_file_type(content: bytes) -> str | None: - if content.startswith(b"\xff\xd8\xff"): - return "image/jpeg" - if content.startswith(b"\x89PNG\r\n\x1a\n"): - return "image/png" - if content.startswith(b"GIF87a") or content.startswith(b"GIF89a"): - return "image/gif" - if content.startswith(b"PK\x03\x04"): - if b"[Content_Types].xml" in content[:1000]: - return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - return "application/zip" - if content.startswith(b"%PDF"): - return "application/pdf" - if content.startswith(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"): - return "application/msword" - return None - - @staticmethod - def _generate_http_url(file_path: str, base_url: str | None = None) -> str: - static_root = str(STATIC_DIR) - try: - relative_path = os.path.relpath(file_path, static_root) - url_path = relative_path.replace(os.sep, "/") - except ValueError: - url_path = os.path.basename(file_path) - - if base_url: - base_part = base_url.rstrip("/") - static_part = settings.STATIC_URL.lstrip("/") - file_part = url_path.lstrip("/") - http_url = f"{base_part}/{static_part}/{file_part}".replace("//", "/").replace(":/", "://") - else: - http_url = f"{settings.STATIC_URL}/{url_path}".replace("//", "/") - - return http_url - - @staticmethod - def _get_file_info(file_path: str, base_url: str | None = None) -> ResourceItemSchema | None: - try: - safe_path = file_path - if not os.path.exists(safe_path): - return None - - stat = os.stat(safe_path) - path_obj = Path(safe_path) - resource_root = ResourceService._get_resource_root() - - try: - relative_path = os.path.relpath(safe_path, resource_root) - except ValueError: - relative_path = os.path.basename(safe_path) - - http_url = ResourceService._generate_http_url(safe_path, base_url) - is_hidden = path_obj.name.startswith(".") - - return ResourceItemSchema( - name=path_obj.name, - file_url=http_url, - relative_path=relative_path, - is_file=os.path.isfile(safe_path), - is_dir=os.path.isdir(safe_path), - size=stat.st_size if os.path.isfile(safe_path) else None, - created_time=datetime.fromtimestamp(stat.st_ctime), - modified_time=datetime.fromtimestamp(stat.st_mtime), - is_hidden=is_hidden, - ) - except Exception as e: - logger.error(f"获取文件信息失败: {e!s}") - return None - - @staticmethod - async def get_resources_list( - search: ResourceSearchQueryParam | None = None, - order_by: str | None = None, - base_url: str | None = None, - ) -> list[ResourceItemSchema]: - try: - if search and hasattr(search, "path") and search.path and isinstance(search.path, str): - resource_root = ResourceService._get_safe_path(search.path) - else: - resource_root = ResourceService._get_resource_root() - - if not os.path.exists(resource_root): - raise CustomException(msg="目录不存在") - - if not os.path.isdir(resource_root): - raise CustomException(msg="路径不是目录") - - all_resources = [] - - try: - include_hidden = search.include_hidden if search and hasattr(search, "include_hidden") else False - - for item_name in os.listdir(resource_root): - if item_name.startswith(".") and not include_hidden: - continue - - item_path = os.path.join(resource_root, item_name) - file_info = ResourceService._get_file_info(item_path, base_url) - - if file_info: - if search and hasattr(search, "name") and search.name and search.name[1]: - search_keyword = search.name[1].lower() - if search_keyword not in file_info.name.lower(): - continue - all_resources.append(file_info) - - except PermissionError: - raise CustomException(msg="没有权限访问此目录") - - sorted_resources = ResourceService._sort_results(all_resources, order_by) - - if len(sorted_resources) > ResourceService.MAX_SEARCH_RESULTS: - sorted_resources = sorted_resources[: ResourceService.MAX_SEARCH_RESULTS] - - return sorted_resources - - except Exception as e: - logger.error(f"搜索资源失败: {e!s}") - raise CustomException(msg=f"搜索资源失败: {e!s}") - - @staticmethod - async def export_resource(data_list: list[ResourceItemSchema]) -> bytes: - mapping_dict = { - "name": "文件名", - "path": "文件路径", - "size": "文件大小", - "created_time": "创建时间", - "modified_time": "修改时间", - "parent_path": "父目录", - } - - export_data = [item.model_dump() for item in data_list] - - for item in export_data: - if item.get("size"): - item["size"] = ResourceService._format_file_size(item["size"]) - - return ExcelUtil.export_list2excel(list_data=export_data, mapping_dict=mapping_dict) - - @staticmethod - async def download_file(file_path: str) -> str: - safe_path = ResourceService._get_safe_path(file_path) - if not os.path.exists(safe_path): - raise CustomException(msg="文件不存在") - if not os.path.isfile(safe_path): - raise CustomException(msg="路径不是文件") - return safe_path - - @staticmethod - async def delete_file(paths: list[str]) -> None: - for path_item in paths: - safe_path = ResourceService._get_safe_path(path_item) - - if not os.path.exists(safe_path): - raise CustomException(msg=f"文件不存在: {path_item}") - - try: - if os.path.isfile(safe_path): - os.remove(safe_path) - elif os.path.isdir(safe_path): - shutil.rmtree(safe_path) - else: - raise CustomException(msg=f"无法识别的文件类型: {path_item}") - except PermissionError: - raise CustomException(msg=f"没有权限删除: {path_item}") - except OSError as e: - raise CustomException(msg=f"删除失败: {path_item} - {e!s}") - - logger.info(f"成功删除: {path_item}") - - @staticmethod - async def move_file(data: ResourceMoveSchema) -> None: - source_safe = ResourceService._get_safe_path(data.source_path) - target_dir_safe = ResourceService._get_safe_path(data.target_path) - - if not os.path.exists(source_safe): - raise CustomException(msg=f"源文件不存在: {data.source_path}") - - if not os.path.isdir(target_dir_safe): - raise CustomException(msg=f"目标目录不存在: {data.target_path}") - - filename = os.path.basename(source_safe) - target_path = os.path.join(target_dir_safe, filename) - - if os.path.exists(target_path): - raise CustomException(msg=f"目标位置已存在同名文件: {filename}") - - try: - shutil.move(source_safe, target_path) - except PermissionError: - raise CustomException(msg=f"没有权限移动文件: {data.source_path}") - except OSError as e: - raise CustomException(msg=f"移动文件失败: {e!s}") - - logger.info(f"成功移动文件: {data.source_path} -> {data.target_path}") - - @staticmethod - async def copy_file(data: ResourceCopySchema) -> None: - source_safe = ResourceService._get_safe_path(data.source_path) - target_dir_safe = ResourceService._get_safe_path(data.target_path) - - if not os.path.exists(source_safe): - raise CustomException(msg=f"源文件不存在: {data.source_path}") - - if not os.path.isdir(target_dir_safe): - raise CustomException(msg=f"目标目录不存在: {data.target_path}") - - filename = os.path.basename(source_safe) - target_path = os.path.join(target_dir_safe, filename) - - if os.path.exists(target_path): - raise CustomException(msg=f"目标位置已存在同名文件: {filename}") - - try: - if os.path.isdir(source_safe): - shutil.copytree(source_safe, target_path) - else: - shutil.copy2(source_safe, target_path) - except PermissionError: - raise CustomException(msg=f"没有权限复制文件: {data.source_path}") - except OSError as e: - raise CustomException(msg=f"复制文件失败: {e!s}") - - logger.info(f"成功复制文件: {data.source_path} -> {data.target_path}") - - @staticmethod - async def rename_file(data: ResourceRenameSchema) -> None: - safe_path = ResourceService._get_safe_path(data.old_path) - parent_dir = os.path.dirname(safe_path) - safe_name = ResourceService._sanitize_filename(data.new_name) - - new_path = os.path.join(parent_dir, safe_name) - - if os.path.exists(new_path): - raise CustomException(msg=f"目标文件名已存在: {safe_name}") - - try: - os.rename(safe_path, new_path) - except PermissionError: - raise CustomException(msg=f"没有权限重命名: {data.old_path}") - except OSError as e: - raise CustomException(msg=f"重命名失败: {e!s}") - - logger.info(f"成功重命名: {data.old_path} -> {safe_name}") - - @staticmethod - async def create_directory(data: ResourceCreateDirSchema) -> None: - parent_dir = ResourceService._get_safe_path(data.parent_path) - - if not os.path.isdir(parent_dir): - raise CustomException(msg=f"父目录不存在: {data.parent_path}") - - safe_name = ResourceService._sanitize_filename(data.dir_name) - new_dir = os.path.join(parent_dir, safe_name) - - if os.path.exists(new_dir): - raise CustomException(msg=f"目录已存在: {data.dir_name}") - - try: - os.makedirs(new_dir, exist_ok=False) - except PermissionError: - raise CustomException(msg=f"没有权限创建目录: {data.dir_name}") - except OSError as e: - raise CustomException(msg=f"创建目录失败: {e!s}") - - logger.info(f"成功创建目录: {data.parent_path}/{safe_name}") - - @staticmethod - async def _get_directory_stats(path: str, include_hidden: bool = False) -> dict[str, int]: - stats = {"files": 0, "dirs": 0, "size": 0} - - try: - for root, dirs, files in os.walk(path): - if not include_hidden: - dirs[:] = [d for d in dirs if not d.startswith(".")] - files = [f for f in files if not f.startswith(".")] - - stats["dirs"] += len(dirs) - stats["files"] += len(files) - - for file in files: - file_path = os.path.join(root, file) - try: - stats["size"] += os.path.getsize(file_path) - except OSError: - continue - except Exception: - pass - - return stats - - @staticmethod - def _sort_results(results: list[ResourceItemSchema], order_by: str | None = None) -> list[ResourceItemSchema]: - try: - if not order_by: - return sorted(results, key=lambda x: x.name, reverse=False) - - sort_conditions = ast.literal_eval(order_by) - if isinstance(sort_conditions, list): - - def sort_key(item): - keys = [] - for cond in sort_conditions: - field = cond.get("field", "name") - value = getattr(item, field, "") - if field in ["created_time", "modified_time", "accessed_time"] and value: - if isinstance(value, str): - value = datetime.fromisoformat(value) - keys.append(value) - return keys - - reverse = False - if sort_conditions and isinstance(sort_conditions[0], dict): - order = sort_conditions[0].get("order", "asc") - reverse = order.lower() == "desc" - - return sorted(results, key=sort_key, reverse=reverse) - - return sorted(results, key=lambda x: x.name, reverse=False) - - except (ValueError, SyntaxError): - return sorted(results, key=lambda x: x.name, reverse=False) - - @staticmethod - def _format_file_size(size_bytes: int) -> str: - size = float(size_bytes) - for unit in ["B", "KB", "MB", "GB"]: - if size < 1024: - return f"{size:.2f} {unit}" - size /= 1024 - return f"{size:.2f} TB" diff --git a/backend/app/api/v1/module_monitor/server/service.py b/backend/app/api/v1/module_monitor/server/service.py index ba6a2372..d19baf0c 100644 --- a/backend/app/api/v1/module_monitor/server/service.py +++ b/backend/app/api/v1/module_monitor/server/service.py @@ -1,3 +1,4 @@ +import asyncio import platform import socket import time @@ -22,6 +23,12 @@ class ServerService: @staticmethod async def get_server_monitor_info() -> ServerMonitorSchema: + # 采样全部为同步阻塞调用（psutil.disk_usage 在网络盘/卸载中的挂载点上 + # 可达秒级，socket.gethostbyname 依赖 DNS），放到工作线程执行避免拖慢事件循环 + return await asyncio.to_thread(ServerService._collect_monitor_info) + + @staticmethod + def _collect_monitor_info() -> ServerMonitorSchema: return ServerMonitorSchema( cpu=ServerService._get_cpu_info(), mem=ServerService._get_memory_info(), diff --git a/backend/app/api/v1/module_storage/__init__.py b/backend/app/api/v1/module_storage/__init__.py deleted file mode 100644 index 7e10a755..00000000 --- a/backend/app/api/v1/module_storage/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from fastapi import APIRouter - -from app.api.v1.module_storage.file.controller import StorageFileRouter -from app.api.v1.module_storage.source.controller import StorageSourceRouter -from app.api.v1.module_storage.transfer.controller import StorageTransferRouter - -storage_router = APIRouter(prefix="/storage") - -storage_router.include_router(StorageSourceRouter) -storage_router.include_router(StorageFileRouter) -storage_router.include_router(StorageTransferRouter) diff --git a/backend/app/api/v1/module_storage/core/base.py b/backend/app/api/v1/module_storage/core/base.py deleted file mode 100644 index 53809ab7..00000000 --- a/backend/app/api/v1/module_storage/core/base.py +++ /dev/null @@ -1,94 +0,0 @@ -from abc import ABC, abstractmethod -from datetime import datetime - -from pydantic import BaseModel, Field - -from app.api.v1.module_storage.core.constants import StorageProtocol - - -class StorageAdapterConfig(BaseModel): - """存储适配器配置（从 StorageSourceModel 剥离加密字段后注入，解耦 ORM 与协议层）""" - - protocol: StorageProtocol = Field(description="存储协议") - host: str = Field(description="主机地址") - port: int = Field(description="端口") - username: str | None = Field(default=None, description="用户名/AccessKey") - password: str | None = Field(default=None, description="密码/SecretKey（已解密）") - bucket: str | None = Field(default=None, description="桶名（对象存储）或根目录（FTP/SFTP）") - endpoint: str | None = Field(default=None, description="接入点（对象存储）") - region: str | None = Field(default=None, description="区域（对象存储）") - path_prefix: str | None = Field(default=None, description="统一路径前缀") - is_secure: bool = Field(default=False, description="是否启用 TLS（FTPS）") - implicit_tls: bool = Field(default=False, description="FTPS 是否隐式 TLS（默认显式）") - - @property - def full_prefix(self) -> str: - """规范化后的完整路径前缀（去除首尾斜杠）。""" - prefix = self.path_prefix or "" - return prefix.strip("/") - - -class StorageObject(BaseModel): - """远端文件对象信息""" - - name: str = Field(description="文件/目录名") - key: str = Field(description="相对路径（含前缀时自动剥离）") - is_dir: bool = Field(default=False, description="是否目录") - size: int | None = Field(default=None, description="大小（字节）") - modified_time: datetime | None = Field(default=None, description="修改时间") - - -class BaseStorageAdapter(ABC): - """存储协议适配器抽象基类 - - 所有适配器均在事件循环中通过 ``asyncio.to_thread`` 包装同步 SDK 调用， - 避免阻塞事件循环。适配器实例按请求创建（无连接池复用），用完由调用方关闭。 - """ - - def __init__(self, config: StorageAdapterConfig) -> None: - self.config = config - - def _join_key(self, remote_path: str) -> str: - """将用户传入的远端相对路径拼接路径前缀，得到协议层完整 key。""" - remote_path = remote_path.strip("/") - if self.config.full_prefix: - return f"{self.config.full_prefix}/{remote_path}" - return remote_path - - def _strip_prefix(self, key: str) -> str: - """从协议层完整 key 剥离路径前缀，返回用户可见的相对路径。""" - prefix = self.config.full_prefix - if prefix and key.startswith(f"{prefix}/"): - return key[len(prefix) + 1 :] - return key - - @abstractmethod - async def test_connection(self) -> bool: - """测试连接是否可用。""" - - @abstractmethod - async def upload(self, local_path: str, remote_path: str) -> str: - """上传本地文件到远端，返回远端完整 key。""" - - @abstractmethod - async def download(self, remote_path: str, local_path: str) -> str: - """下载远端文件到本地，返回本地路径。""" - - @abstractmethod - async def delete(self, remote_path: str) -> None: - """删除远端文件（目录递归删除由协议层自行处理）。""" - - @abstractmethod - async def exists(self, remote_path: str) -> bool: - """判断远端文件是否存在。""" - - @abstractmethod - async def list(self, prefix: str = "") -> list[StorageObject]: - """列出远端目录下的文件与目录（不含前缀）。""" - - async def get_url(self, remote_path: str, expire: int = 3600) -> str | None: - """获取访问 URL（对象存储返回预签名 URL；FTP/SFTP 不支持返回 None）。""" - return None - - async def close(self) -> None: # noqa: B027 - 可选钩子，无连接池的协议实现为空操作 - """释放连接资源。""" diff --git a/backend/app/api/v1/module_storage/core/constants.py b/backend/app/api/v1/module_storage/core/constants.py deleted file mode 100644 index 510a4dab..00000000 --- a/backend/app/api/v1/module_storage/core/constants.py +++ /dev/null @@ -1,27 +0,0 @@ -from enum import Enum - - -class StorageProtocol(str, Enum): - """存储协议枚举""" - - FTP = "ftp" - FTPS = "ftps" - SFTP = "sftp" - S3 = "s3" - OBS = "obs" - OSS = "oss" - COS = "cos" - LOCAL = "local" - - -# 各协议默认端口 -DEFAULT_PORTS: dict[StorageProtocol, int] = { - StorageProtocol.FTP: 21, - StorageProtocol.FTPS: 990, - StorageProtocol.SFTP: 22, - StorageProtocol.S3: 443, - StorageProtocol.OBS: 443, - StorageProtocol.OSS: 443, - StorageProtocol.COS: 443, - StorageProtocol.LOCAL: 0, -} diff --git a/backend/app/api/v1/module_storage/core/cos_adapter.py b/backend/app/api/v1/module_storage/core/cos_adapter.py deleted file mode 100644 index dba5f96a..00000000 --- a/backend/app/api/v1/module_storage/core/cos_adapter.py +++ /dev/null @@ -1,141 +0,0 @@ -import asyncio - -from qcloud_cos import CosConfig, CosS3Client - -from app.api.v1.module_storage.core.base import BaseStorageAdapter, StorageObject -from app.api.v1.module_storage.core.constants import StorageProtocol -from app.core.exceptions import CustomException -from app.core.logger import logger - - -class CosStorageAdapter(BaseStorageAdapter): - """腾讯云 COS 存储适配器（cos-python-sdk-v5，同步调用经 asyncio.to_thread 包装）。""" - - protocol = StorageProtocol.COS - - def __init__(self, config) -> None: - super().__init__(config) - if not self.config.region: - raise CustomException(msg="COS 存储源必须配置 region") - if not self.config.username or not self.config.password: - raise CustomException(msg="COS 存储源必须配置 SecretId/SecretKey") - cos_config = CosConfig( - Region=self.config.region, - SecretId=self.config.username or "", - SecretKey=self.config.password or "", - Scheme="https" if self.config.is_secure else "http", - ) - self.client = CosS3Client(cos_config) - - def _require_bucket(self) -> str: - if not self.config.bucket: - raise CustomException(msg="存储源未配置 bucket") - return self.config.bucket - - def _sync_test_connection(self) -> bool: - try: - self.client.list_buckets() - return True - except Exception as e: - logger.warning(f"COS 连接测试失败: {e}") - return False - - def _sync_upload(self, local_path: str, remote_path: str) -> str: - try: - self.client.upload_file( - Bucket=self._require_bucket(), - Key=remote_path, - LocalFilePath=local_path, - EnableMD5=False, - ) - except Exception as e: - raise CustomException(msg=f"COS 上传失败: {e!s}") - return remote_path - - def _sync_download(self, remote_path: str, local_path: str) -> str: - try: - self.client.download_file( - Bucket=self._require_bucket(), - Key=remote_path, - DestFilePath=local_path, - ) - except Exception as e: - raise CustomException(msg=f"COS 下载失败: {e!s}") - return local_path - - def _sync_delete(self, remote_path: str) -> None: - try: - self.client.delete_object(Bucket=self._require_bucket(), Key=remote_path) - except Exception as e: - raise CustomException(msg=f"COS 删除失败: {e!s}") - - def _sync_exists(self, remote_path: str) -> bool: - try: - return self.client.object_exists(Bucket=self._require_bucket(), Key=remote_path) - except Exception: - return False - - def _sync_list(self, prefix: str) -> list[StorageObject]: - try: - resp = self.client.list_objects(Bucket=self._require_bucket(), Prefix=prefix, Delimiter="/") - except Exception as e: - raise CustomException(msg=f"COS 列表失败: {e!s}") - - result: list[StorageObject] = [] - for common in resp.get("CommonPrefixes", []): - raw_key = common.get("Prefix", "").rstrip("/") - result.append(StorageObject(name=raw_key.rsplit("/", 1)[-1], key=self._strip_prefix(raw_key), is_dir=True)) - for obj in resp.get("Contents", []): - raw_key = obj.get("Key", "") - if raw_key == prefix: - continue - result.append( - StorageObject( - name=raw_key.rsplit("/", 1)[-1], - key=self._strip_prefix(raw_key), - is_dir=False, - size=obj.get("Size"), - modified_time=obj.get("LastModified"), - ) - ) - return result - - def _sync_get_url(self, remote_path: str, expire: int) -> str: - try: - return self.client.get_presigned_url( - Method="GET", - Bucket=self._require_bucket(), - Key=remote_path, - Expired=expire, - ) - except Exception as e: - raise CustomException(msg=f"COS 生成预签名 URL 失败: {e!s}") - - # ── 异步公开接口 ──────────────────────────────────────────────── - - async def test_connection(self) -> bool: - return await asyncio.to_thread(self._sync_test_connection) - - async def upload(self, local_path: str, remote_path: str) -> str: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_upload, local_path, full_key) - - async def download(self, remote_path: str, local_path: str) -> str: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_download, full_key, local_path) - - async def delete(self, remote_path: str) -> None: - full_key = self._join_key(remote_path) - await asyncio.to_thread(self._sync_delete, full_key) - - async def exists(self, remote_path: str) -> bool: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_exists, full_key) - - async def list(self, prefix: str = "") -> list[StorageObject]: - full_prefix = self._join_key(prefix) if prefix else self.config.full_prefix - return await asyncio.to_thread(self._sync_list, full_prefix) - - async def get_url(self, remote_path: str, expire: int = 3600) -> str | None: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_get_url, full_key, expire) diff --git a/backend/app/api/v1/module_storage/core/encrypt.py b/backend/app/api/v1/module_storage/core/encrypt.py deleted file mode 100644 index 5a504d4a..00000000 --- a/backend/app/api/v1/module_storage/core/encrypt.py +++ /dev/null @@ -1,35 +0,0 @@ -import base64 -import hashlib - -from cryptography.fernet import Fernet, InvalidToken - -from app.config.setting import settings -from app.core.exceptions import CustomException - -_fernet: Fernet | None = None - - -def _get_fernet() -> Fernet: - """惰性构建 Fernet 实例，密钥由 settings.SECRET_KEY 派生（SHA-256 → url-safe base64）。""" - global _fernet - if _fernet is None: - digest = hashlib.sha256(settings.SECRET_KEY.encode("utf-8")).digest() - _fernet = Fernet(base64.urlsafe_b64encode(digest)) - return _fernet - - -def encrypt_password(plain: str | None) -> str: - """明文密码 → Fernet 密文。空值原样返回空串。""" - if not plain: - return "" - return _get_fernet().encrypt(plain.encode("utf-8")).decode("utf-8") - - -def decrypt_password(cipher: str | None) -> str: - """Fernet 密文 → 明文密码。空值原样返回空串。""" - if not cipher: - return "" - try: - return _get_fernet().decrypt(cipher.encode("utf-8")).decode("utf-8") - except (InvalidToken, ValueError): - raise CustomException(msg="存储源密码解密失败，可能原因：SECRET_KEY 变更或数据损坏") diff --git a/backend/app/api/v1/module_storage/core/obs_adapter.py b/backend/app/api/v1/module_storage/core/obs_adapter.py deleted file mode 100644 index 593190ee..00000000 --- a/backend/app/api/v1/module_storage/core/obs_adapter.py +++ /dev/null @@ -1,160 +0,0 @@ -import asyncio -from datetime import UTC, datetime -from typing import Any - -from obs import ObsClient - -from app.api.v1.module_storage.core.base import BaseStorageAdapter, StorageObject -from app.api.v1.module_storage.core.constants import StorageProtocol -from app.core.exceptions import CustomException -from app.core.logger import logger - - -class ObsStorageAdapter(BaseStorageAdapter): - """华为云 OBS 存储适配器（esdk-obs-python，同步调用经 asyncio.to_thread 包装）。""" - - protocol = StorageProtocol.OBS - - def __init__(self, config) -> None: - super().__init__(config) - if not self.config.endpoint: - raise CustomException(msg="OBS 存储源必须配置 endpoint") - self.client = ObsClient( - access_key_id=self.config.username or "", - secret_access_key=self.config.password or "", - server=self.config.endpoint, - ) - - def _require_bucket(self) -> str: - if not self.config.bucket: - raise CustomException(msg="存储源未配置 bucket") - return self.config.bucket - - @staticmethod - def _is_ok(resp: Any) -> bool: - """判断 OBS 响应是否成功（status < 300）。SDK 未提供类型存根，故用 getattr 访问动态属性。""" - status = getattr(resp, "status", None) - return status is not None and status < 300 - - @staticmethod - def _error_desc(resp: Any) -> str: - """提取 OBS 响应中的错误描述。""" - code = getattr(resp, "errorCode", "") or "" - message = getattr(resp, "errorMessage", "") or "" - return f"{code} {message}".strip() - - def _sync_test_connection(self) -> bool: - try: - resp = self.client.listBuckets() - if self._is_ok(resp): - return True - logger.warning(f"OBS 连接测试失败: {self._error_desc(resp)}") - return False - except Exception as e: - logger.warning(f"OBS 连接测试失败: {e}") - return False - - def _sync_upload(self, local_path: str, remote_path: str) -> str: - try: - resp = self.client.putFile(bucketName=self._require_bucket(), objectKey=remote_path, file_path=local_path) - if not self._is_ok(resp): - raise CustomException(msg=f"OBS 上传失败: {self._error_desc(resp)}") - except CustomException: - raise - except Exception as e: - raise CustomException(msg=f"OBS 上传失败: {e!s}") - return remote_path - - def _sync_download(self, remote_path: str, local_path: str) -> str: - try: - resp = self.client.getObject(bucketName=self._require_bucket(), objectKey=remote_path, downloadPath=local_path) - if not self._is_ok(resp): - raise CustomException(msg=f"OBS 下载失败: {self._error_desc(resp)}") - except CustomException: - raise - except Exception as e: - raise CustomException(msg=f"OBS 下载失败: {e!s}") - return local_path - - def _sync_delete(self, remote_path: str) -> None: - try: - resp = self.client.deleteObject(bucketName=self._require_bucket(), objectKey=remote_path) - if not self._is_ok(resp): - raise CustomException(msg=f"OBS 删除失败: {self._error_desc(resp)}") - except CustomException: - raise - except Exception as e: - raise CustomException(msg=f"OBS 删除失败: {e!s}") - - def _sync_exists(self, remote_path: str) -> bool: - try: - resp = self.client.getObjectMetadata(bucketName=self._require_bucket(), objectKey=remote_path) - return self._is_ok(resp) - except Exception: - return False - - def _sync_list(self, prefix: str) -> list[StorageObject]: - try: - resp = self.client.listObjects(bucketName=self._require_bucket(), prefix=prefix, delimiter="/") - except Exception as e: - raise CustomException(msg=f"OBS 列表失败: {e!s}") - if not self._is_ok(resp): - raise CustomException(msg=f"OBS 列表失败: {self._error_desc(resp)}") - - body = getattr(resp, "body", None) - result: list[StorageObject] = [] - for common in getattr(body, "commonPrefixes", None) or []: - raw_key = getattr(common, "prefix", "") or "" - raw_key = raw_key.rstrip("/") - result.append(StorageObject(name=raw_key.rsplit("/", 1)[-1], key=self._strip_prefix(raw_key), is_dir=True)) - for obj in getattr(body, "contents", None) or []: - raw_key = getattr(obj, "key", "") or "" - if raw_key == prefix: - continue - last_modified = getattr(obj, "lastModified", None) - result.append( - StorageObject( - name=raw_key.rsplit("/", 1)[-1], - key=self._strip_prefix(raw_key), - is_dir=False, - size=getattr(obj, "size", None), - modified_time=datetime.fromtimestamp(last_modified / 1000, tz=UTC) if last_modified else None, - ) - ) - return result - - def _sync_get_url(self, remote_path: str, expire: int) -> str: - try: - resp = self.client.createSignedUrl("GET", bucketName=self._require_bucket(), objectKey=remote_path, expires=expire) - return getattr(resp, "signedUrl", "") - except Exception as e: - raise CustomException(msg=f"OBS 生成预签名 URL 失败: {e!s}") - - # ── 异步公开接口 ──────────────────────────────────────────────── - - async def test_connection(self) -> bool: - return await asyncio.to_thread(self._sync_test_connection) - - async def upload(self, local_path: str, remote_path: str) -> str: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_upload, local_path, full_key) - - async def download(self, remote_path: str, local_path: str) -> str: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_download, full_key, local_path) - - async def delete(self, remote_path: str) -> None: - full_key = self._join_key(remote_path) - await asyncio.to_thread(self._sync_delete, full_key) - - async def exists(self, remote_path: str) -> bool: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_exists, full_key) - - async def list(self, prefix: str = "") -> list[StorageObject]: - full_prefix = self._join_key(prefix) if prefix else self.config.full_prefix - return await asyncio.to_thread(self._sync_list, full_prefix) - - async def get_url(self, remote_path: str, expire: int = 3600) -> str | None: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_get_url, full_key, expire) diff --git a/backend/app/api/v1/module_storage/core/oss_adapter.py b/backend/app/api/v1/module_storage/core/oss_adapter.py deleted file mode 100644 index 4c2c78cc..00000000 --- a/backend/app/api/v1/module_storage/core/oss_adapter.py +++ /dev/null @@ -1,152 +0,0 @@ -import asyncio -from datetime import UTC, datetime, timedelta - -import alibabacloud_oss_v2 as oss - -from app.api.v1.module_storage.core.base import BaseStorageAdapter, StorageObject -from app.api.v1.module_storage.core.constants import StorageProtocol -from app.core.exceptions import CustomException -from app.core.logger import logger - - -class OssStorageAdapter(BaseStorageAdapter): - """阿里云 OSS 存储适配器（alibabacloud_oss_v2 SDK，同步调用经 asyncio.to_thread 包装）。""" - - protocol = StorageProtocol.OSS - - def __init__(self, config) -> None: - super().__init__(config) - if not self.config.endpoint: - raise CustomException(msg="OSS 存储源必须配置 endpoint") - if not self.config.region: - raise CustomException(msg="OSS 存储源必须配置 region（V4 签名要求，如 cn-hangzhou）") - if not self.config.username or not self.config.password: - raise CustomException(msg="OSS 存储源必须配置 AccessKeyId 与 AccessKeySecret") - self.bucket_name = self.config.bucket or "" - cfg = oss.config.load_default() - cfg.credentials_provider = oss.credentials.StaticCredentialsProvider(self.config.username, self.config.password) - cfg.region = self.config.region - cfg.endpoint = self.config.endpoint - self.client = oss.Client(cfg) - - def _sync_test_connection(self) -> bool: - try: - self.client.get_bucket_info(oss.GetBucketInfoRequest(bucket=self.bucket_name)) - return True - except oss.exceptions.ServiceError as e: - logger.warning(f"OSS 连接测试失败: {e.code} {e.message}") - return False - except Exception as e: - logger.warning(f"OSS 连接测试失败: {e}") - return False - - def _sync_upload(self, local_path: str, remote_path: str) -> str: - try: - self.client.put_object_from_file( - oss.PutObjectRequest(bucket=self.bucket_name, key=remote_path), - local_path, - ) - except Exception as e: - raise CustomException(msg=f"OSS 上传失败: {e!s}") - return remote_path - - def _sync_download(self, remote_path: str, local_path: str) -> str: - try: - self.client.get_object_to_file( - oss.GetObjectRequest(bucket=self.bucket_name, key=remote_path), - local_path, - ) - except Exception as e: - raise CustomException(msg=f"OSS 下载失败: {e!s}") - return local_path - - def _sync_delete(self, remote_path: str) -> None: - try: - self.client.delete_object(oss.DeleteObjectRequest(bucket=self.bucket_name, key=remote_path)) - except Exception as e: - raise CustomException(msg=f"OSS 删除失败: {e!s}") - - def _sync_exists(self, remote_path: str) -> bool: - try: - self.client.head_object(oss.HeadObjectRequest(bucket=self.bucket_name, key=remote_path)) - return True - except oss.exceptions.ServiceError as e: - if e.status_code == 404: - return False - logger.warning(f"OSS 判断文件存在失败: {e.code} {e.message}") - return False - except Exception: - return False - - @staticmethod - def _to_utc_dt(value: int | float | datetime | None) -> datetime | None: - """兼容 SDK 返回的时间戳（int/float）与 datetime 两种类型。""" - if value is None: - return None - if isinstance(value, datetime): - return value.astimezone(UTC) if value.tzinfo else value.replace(tzinfo=UTC) - return datetime.fromtimestamp(value, tz=UTC) - - def _sync_list(self, prefix: str) -> list[StorageObject]: - try: - paginator = self.client.list_objects_v2_paginator() - result: list[StorageObject] = [] - for page in paginator.iter_page(oss.ListObjectsV2Request(bucket=self.bucket_name, prefix=prefix, delimiter="/")): - for common in page.common_prefixes or []: - raw_key = (common.prefix or "").rstrip("/") - result.append(StorageObject(name=raw_key.rsplit("/", 1)[-1], key=self._strip_prefix(raw_key), is_dir=True)) - for obj in page.contents or []: - raw_key = obj.key or "" - if raw_key == prefix: - continue - result.append( - StorageObject( - name=raw_key.rsplit("/", 1)[-1], - key=self._strip_prefix(raw_key), - is_dir=False, - size=obj.size, - modified_time=self._to_utc_dt(obj.last_modified), - ) - ) - return result - except Exception as e: - raise CustomException(msg=f"OSS 列表失败: {e!s}") - - def _sync_get_url(self, remote_path: str, expire: int) -> str | None: - try: - result = self.client.presign( - oss.GetObjectRequest(bucket=self.bucket_name, key=remote_path), - expires=timedelta(seconds=expire), - ) - return result.url - except Exception as e: - raise CustomException(msg=f"OSS 生成预签名 URL 失败: {e!s}") - - # ── 异步公开接口 ──────────────────────────────────────────────── - - async def test_connection(self) -> bool: - return await asyncio.to_thread(self._sync_test_connection) - - async def upload(self, local_path: str, remote_path: str) -> str: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_upload, local_path, full_key) - - async def download(self, remote_path: str, local_path: str) -> str: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_download, full_key, local_path) - - async def delete(self, remote_path: str) -> None: - full_key = self._join_key(remote_path) - await asyncio.to_thread(self._sync_delete, full_key) - - async def exists(self, remote_path: str) -> bool: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_exists, full_key) - - async def list(self, prefix: str = "") -> list[StorageObject]: - full_prefix = self._join_key(prefix) if prefix else self.config.full_prefix - return await asyncio.to_thread(self._sync_list, full_prefix) - - async def get_url(self, remote_path: str, expire: int = 3600) -> str | None: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_get_url, full_key, expire) diff --git a/backend/app/api/v1/module_storage/core/s3_adapter.py b/backend/app/api/v1/module_storage/core/s3_adapter.py deleted file mode 100644 index 5e8250bd..00000000 --- a/backend/app/api/v1/module_storage/core/s3_adapter.py +++ /dev/null @@ -1,149 +0,0 @@ -import asyncio - -import boto3 -from botocore.exceptions import ClientError - -from app.api.v1.module_storage.core.base import BaseStorageAdapter, StorageObject -from app.api.v1.module_storage.core.constants import StorageProtocol -from app.core.exceptions import CustomException -from app.core.logger import logger - - -class S3StorageAdapter(BaseStorageAdapter): - """S3 兼容对象存储适配器（boto3，同步调用经 asyncio.to_thread 包装）。""" - - protocol = StorageProtocol.S3 - - def __init__(self, config) -> None: - super().__init__(config) - # 凭据为空时传空串而非 None，避免 boto3 走 EC2 实例元数据(IMDS)探测导致超时 - self.client = boto3.client( - "s3", - endpoint_url=self.config.endpoint, - region_name=self.config.region, - aws_access_key_id=self.config.username or "", - aws_secret_access_key=self.config.password or "", - ) - - def _require_bucket(self) -> str: - if not self.config.bucket: - raise CustomException(msg="存储源未配置 bucket") - return self.config.bucket - - def _sync_test_connection(self) -> bool: - try: - self.client.head_bucket(Bucket=self._require_bucket()) - return True - except ClientError as e: - code = e.response.get("Error", {}).get("Code", "") - # 403 表示凭据有效但无权限查看桶，连接本身是通的 - if code == "403": - return True - logger.warning(f"S3 连接测试失败: {code} {e}") - return False - except Exception as e: - logger.warning(f"S3 连接测试失败: {e}") - return False - - def _sync_upload(self, local_path: str, remote_path: str) -> str: - try: - self.client.upload_file(local_path, self._require_bucket(), remote_path) - except Exception as e: - raise CustomException(msg=f"S3 上传失败: {e!s}") - return remote_path - - def _sync_download(self, remote_path: str, local_path: str) -> str: - try: - self.client.download_file(self._require_bucket(), remote_path, local_path) - except Exception as e: - raise CustomException(msg=f"S3 下载失败: {e!s}") - return local_path - - def _sync_delete(self, remote_path: str) -> None: - try: - self.client.delete_object(Bucket=self._require_bucket(), Key=remote_path) - except Exception as e: - raise CustomException(msg=f"S3 删除失败: {e!s}") - - def _sync_exists(self, remote_path: str) -> bool: - try: - self.client.head_object(Bucket=self._require_bucket(), Key=remote_path) - return True - except ClientError as e: - if e.response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 404: - return False - logger.warning(f"S3 head_object 失败: {e}") - return False - except Exception: - return False - - def _sync_list(self, prefix: str) -> list[StorageObject]: - try: - resp = self.client.list_objects_v2(Bucket=self._require_bucket(), Prefix=prefix, Delimiter="/") - except Exception as e: - raise CustomException(msg=f"S3 列表失败: {e!s}") - - result: list[StorageObject] = [] - for cp in resp.get("CommonPrefixes", []): - raw_key = cp.get("Prefix", "").rstrip("/") - name = raw_key.rsplit("/", 1)[-1] - result.append(StorageObject(name=name, key=self._strip_prefix(raw_key), is_dir=True)) - for obj in resp.get("Contents", []): - raw_key = obj.get("Key", "") - if raw_key == prefix: - continue - name = raw_key.rsplit("/", 1)[-1] - result.append( - StorageObject( - name=name, - key=self._strip_prefix(raw_key), - is_dir=False, - size=obj.get("Size"), - modified_time=obj.get("LastModified"), - ) - ) - return result - - def _sync_get_url(self, remote_path: str, expire: int) -> str: - try: - return self.client.generate_presigned_url( - "get_object", - Params={"Bucket": self._require_bucket(), "Key": remote_path}, - ExpiresIn=expire, - ) - except Exception as e: - raise CustomException(msg=f"S3 生成预签名 URL 失败: {e!s}") - - # ── 异步公开接口 ──────────────────────────────────────────────── - - async def test_connection(self) -> bool: - return await asyncio.to_thread(self._sync_test_connection) - - async def upload(self, local_path: str, remote_path: str) -> str: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_upload, local_path, full_key) - - async def download(self, remote_path: str, local_path: str) -> str: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_download, full_key, local_path) - - async def delete(self, remote_path: str) -> None: - full_key = self._join_key(remote_path) - await asyncio.to_thread(self._sync_delete, full_key) - - async def exists(self, remote_path: str) -> bool: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_exists, full_key) - - async def list(self, prefix: str = "") -> list[StorageObject]: - full_prefix = self._join_key(prefix) if prefix else self.config.full_prefix - return await asyncio.to_thread(self._sync_list, full_prefix) - - async def get_url(self, remote_path: str, expire: int = 3600) -> str | None: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_get_url, full_key, expire) - - async def close(self) -> None: - close = getattr(self.client, "close", None) - if callable(close): - await asyncio.to_thread(close) diff --git a/backend/app/api/v1/module_storage/core/sftp_adapter.py b/backend/app/api/v1/module_storage/core/sftp_adapter.py deleted file mode 100644 index d6db6c23..00000000 --- a/backend/app/api/v1/module_storage/core/sftp_adapter.py +++ /dev/null @@ -1,143 +0,0 @@ -import asyncio -from datetime import UTC, datetime - -import paramiko - -from app.api.v1.module_storage.core.base import BaseStorageAdapter, StorageObject -from app.api.v1.module_storage.core.constants import StorageProtocol -from app.core.exceptions import CustomException -from app.core.logger import logger - - -class SftpStorageAdapter(BaseStorageAdapter): - """SFTP 存储适配器（paramiko，同步调用经 asyncio.to_thread 包装）。""" - - protocol = StorageProtocol.SFTP - - def _new_client(self) -> paramiko.SFTPClient: - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect( - hostname=self.config.host, - port=self.config.port, - username=self.config.username or "", - password=self.config.password or "", - timeout=30, - banner_timeout=30, - ) - return ssh.open_sftp() - - @staticmethod - def _ensure_remote_dir(client: paramiko.SFTPClient, remote_dir: str) -> None: - """递归创建远端目录（mkdir -p）。""" - parts = [p for p in remote_dir.split("/") if p] - current = "" - for part in parts: - current = f"{current}/{part}" if current else part - try: - client.stat(current) - except FileNotFoundError: - client.mkdir(current) - except OSError: - pass - - def _sync_test_connection(self) -> bool: - try: - client = self._new_client() - client.listdir(".") - client.close() - return True - except Exception as e: - logger.warning(f"SFTP 连接测试失败: {e}") - return False - - def _sync_upload(self, local_path: str, remote_path: str) -> str: - client = self._new_client() - try: - dir_part, _ = remote_path.rsplit("/", 1) if "/" in remote_path else ("", remote_path) - if dir_part: - self._ensure_remote_dir(client, dir_part) - client.put(local_path, remote_path) - except Exception as e: - raise CustomException(msg=f"SFTP 上传失败: {e!s}") - finally: - client.close() - return remote_path - - def _sync_download(self, remote_path: str, local_path: str) -> str: - client = self._new_client() - try: - client.get(remote_path, local_path) - except Exception as e: - raise CustomException(msg=f"SFTP 下载失败: {e!s}") - finally: - client.close() - return local_path - - def _sync_delete(self, remote_path: str) -> None: - client = self._new_client() - try: - client.remove(remote_path) - except Exception as e: - raise CustomException(msg=f"SFTP 删除失败: {e!s}") - finally: - client.close() - - def _sync_exists(self, remote_path: str) -> bool: - try: - client = self._new_client() - try: - client.stat(remote_path) - return True - except FileNotFoundError: - return False - finally: - client.close() - except Exception: - return False - - def _sync_list(self, prefix: str) -> list[StorageObject]: - client = self._new_client() - try: - attrs = client.listdir_attr(prefix) - result: list[StorageObject] = [] - for attr in attrs: - result.append( - StorageObject( - name=attr.filename, - key=self._strip_prefix(f"{prefix}/{attr.filename}".strip("/")) if prefix else attr.filename, - is_dir=bool(attr.st_mode and (attr.st_mode & 0o40000)), - size=attr.st_size, - modified_time=datetime.fromtimestamp(attr.st_mtime, tz=UTC) if attr.st_mtime else None, - ) - ) - return result - except Exception as e: - raise CustomException(msg=f"SFTP 列表失败: {e!s}") - finally: - client.close() - - # ── 异步公开接口 ──────────────────────────────────────────────── - - async def test_connection(self) -> bool: - return await asyncio.to_thread(self._sync_test_connection) - - async def upload(self, local_path: str, remote_path: str) -> str: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_upload, local_path, full_key) - - async def download(self, remote_path: str, local_path: str) -> str: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_download, full_key, local_path) - - async def delete(self, remote_path: str) -> None: - full_key = self._join_key(remote_path) - await asyncio.to_thread(self._sync_delete, full_key) - - async def exists(self, remote_path: str) -> bool: - full_key = self._join_key(remote_path) - return await asyncio.to_thread(self._sync_exists, full_key) - - async def list(self, prefix: str = "") -> list[StorageObject]: - full_prefix = self._join_key(prefix) if prefix else self.config.full_prefix - return await asyncio.to_thread(self._sync_list, full_prefix) diff --git a/backend/app/api/v1/module_storage/file/controller.py b/backend/app/api/v1/module_storage/file/controller.py deleted file mode 100644 index ed39be29..00000000 --- a/backend/app/api/v1/module_storage/file/controller.py +++ /dev/null @@ -1,103 +0,0 @@ -import os -from typing import Annotated - -from fastapi import APIRouter, BackgroundTasks, Body, Depends, File, Form, Query, Security, UploadFile -from fastapi.responses import JSONResponse -from sqlalchemy.ext.asyncio import AsyncSession - -from app.api.v1.module_storage.core.base import StorageObject -from app.common.response import ResponseSchema, SuccessResponse, UploadFileResponse -from app.core.base_schema import AuthSchema -from app.core.dependencies import AuthPermission, db_getter -from app.core.router_class import OperationLogRoute - -from .service import StorageFileService - -StorageFileRouter = APIRouter(route_class=OperationLogRoute, prefix="/file", tags=["存储文件"]) - - -def _delete_temp_file(path: str) -> None: - """响应发送后清理临时下载文件。""" - try: - os.unlink(path) - except OSError: - pass - - -@StorageFileRouter.post("/upload", summary="上传文件到存储源", response_model=ResponseSchema[dict]) -async def upload_storage_file_controller( - auth: Annotated[AuthSchema, Security(AuthPermission(["module_storage:file:upload"]))], - db: Annotated[AsyncSession, Depends(db_getter)], - file: Annotated[UploadFile, File(description="上传文件")], - source_id: Annotated[int | None, Form(description="存储源ID（不传使用默认存储源）")] = None, - remote_path: Annotated[str | None, Form(description="远端目录路径（不传自动生成文件名）")] = None, -) -> JSONResponse: - result = await StorageFileService(auth, db).upload(source_id=source_id, file=file, remote_path=remote_path) - return SuccessResponse(data=result, msg="上传文件成功") - - -@StorageFileRouter.post("/download", summary="下载存储源文件", response_model=None) -async def download_storage_file_controller( - auth: Annotated[AuthSchema, Security(AuthPermission(["module_storage:file:download"]))], - db: Annotated[AsyncSession, Depends(db_getter)], - background_tasks: BackgroundTasks, - remote_path: Annotated[str, Body(description="远端文件路径")], - source_id: Annotated[int | None, Body(description="存储源ID（不传使用默认存储源）")] = None, -) -> UploadFileResponse: - local_path, file_name = await StorageFileService(auth, db).download(source_id=source_id, remote_path=remote_path) - background_tasks.add_task(_delete_temp_file, local_path) - return UploadFileResponse(file_path=local_path, filename=file_name) - - -@StorageFileRouter.delete("/delete", summary="删除存储源文件", response_model=ResponseSchema[None]) -async def delete_storage_file_controller( - auth: Annotated[AuthSchema, Security(AuthPermission(["module_storage:file:delete"]))], - db: Annotated[AsyncSession, Depends(db_getter)], - remote_path: Annotated[str, Body(description="远端文件路径")], - source_id: Annotated[int | None, Body(description="存储源ID（不传使用默认存储源）")] = None, -) -> JSONResponse: - await StorageFileService(auth, db).delete(source_id=source_id, remote_path=remote_path) - return SuccessResponse(msg="删除文件成功") - - -@StorageFileRouter.get("/list", summary="查询存储源文件列表", response_model=ResponseSchema[list[StorageObject]]) -async def list_storage_file_controller( - auth: Annotated[AuthSchema, Security(AuthPermission(["module_storage:file:query"]))], - db: Annotated[AsyncSession, Depends(db_getter)], - source_id: Annotated[int | None, Query(description="存储源ID（不传使用默认存储源）")] = None, - prefix: Annotated[str | None, Query(description="目录前缀（可选）")] = None, -) -> JSONResponse: - result = await StorageFileService(auth, db).list(source_id=source_id, prefix=prefix or "") - return SuccessResponse(data=result, msg="查询文件列表成功") - - -@StorageFileRouter.get("/url", summary="获取文件访问URL", response_model=ResponseSchema[str | None]) -async def get_storage_file_url_controller( - auth: Annotated[AuthSchema, Security(AuthPermission(["module_storage:file:query"]))], - db: Annotated[AsyncSession, Depends(db_getter)], - remote_path: Annotated[str, Query(description="远端文件路径")], - source_id: Annotated[int | None, Query(description="存储源ID（不传使用默认存储源）")] = None, - expire: Annotated[int, Query(description="有效期（秒）", ge=60, le=86400)] = 3600, -) -> JSONResponse: - result = await StorageFileService(auth, db).get_url(source_id=source_id, remote_path=remote_path, expire=expire) - return SuccessResponse(data=result, msg="获取文件URL成功") - - -@StorageFileRouter.post("/copy", summary="复制/移动文件", response_model=ResponseSchema[dict]) -async def copy_or_move_storage_file_controller( - auth: Annotated[AuthSchema, Security(AuthPermission(["module_storage:file:update"]))], - db: Annotated[AsyncSession, Depends(db_getter)], - source_id: Annotated[int | None, Body(description="源存储源ID（不传使用默认存储源）")] = None, - source_path: Annotated[str, Body(description="源文件路径")] = "", - target_id: Annotated[int, Body(description="目标存储源ID")] = 0, - target_path: Annotated[str, Body(description="目标路径")] = "", - move: Annotated[bool, Body(description="是否为移动（true 移动/重命名，false 复制）")] = False, -) -> JSONResponse: - result = await StorageFileService(auth, db).copy_or_move( - source_id=source_id, - source_path=source_path, - target_id=target_id, - target_path=target_path, - move=move, - ) - return SuccessResponse(data=result, msg="操作文件成功") diff --git a/backend/app/api/v1/module_storage/file/service.py b/backend/app/api/v1/module_storage/file/service.py deleted file mode 100644 index e82ad7b5..00000000 --- a/backend/app/api/v1/module_storage/file/service.py +++ /dev/null @@ -1,203 +0,0 @@ -import os -import tempfile -from pathlib import Path - -import aiofiles -from fastapi import UploadFile -from sqlalchemy.ext.asyncio import AsyncSession - -from app.api.v1.module_storage.core.base import StorageAdapterConfig, StorageObject -from app.api.v1.module_storage.core.constants import StorageProtocol -from app.api.v1.module_storage.core.encrypt import decrypt_password -from app.api.v1.module_storage.core.factory import StorageAdapterFactory -from app.api.v1.module_storage.source.service import StorageSourceService -from app.core.base_schema import AuthSchema -from app.core.exceptions import CustomException -from app.utils.upload_util import UploadUtil - - -class StorageFileService: - """存储文件操作服务（上传/下载/删除/列表/预签名URL）""" - - def __init__(self, auth: AuthSchema, db: AsyncSession) -> None: - self.auth = auth - self.db = db - - # ── 内部工具 ──────────────────────────────────────────────────── - - @staticmethod - def _validate_remote_path(remote_path: str) -> str: - """规范化并校验远端相对路径（禁止路径穿越）。""" - if not remote_path or not remote_path.strip(): - raise CustomException(msg="请提供文件路径") - parts = [p for p in remote_path.replace("\\", "/").split("/") if p not in ("", ".")] - if any(p == ".." for p in parts) or "\x00" in remote_path: - raise CustomException(msg="非法的文件路径") - return "/".join(parts) - - async def _get_source(self, source_id: int | None) -> StorageAdapterConfig: - """获取存储源并构造适配器配置（密码已解密）。""" - source = await StorageSourceService(self.auth, self.db).get_active_source(source_id) - return StorageAdapterConfig( - protocol=StorageProtocol(source.protocol), - host=source.host, - port=source.port, - username=source.username, - password=decrypt_password(source.password), - bucket=source.bucket, - endpoint=source.endpoint, - region=source.region, - path_prefix=source.path_prefix, - is_secure=source.is_secure, - implicit_tls=source.implicit_tls, - ) - - @staticmethod - async def _save_to_temp(file: UploadFile, suffix: str = "") -> str: - """将上传文件内容落盘到系统临时目录，返回临时路径。""" - fd, path = tempfile.mkstemp(suffix=suffix) - os.close(fd) - try: - async with aiofiles.open(path, "wb") as f: - while chunk := await file.read(1024 * 1024): - await f.write(chunk) - except Exception: - os.unlink(path) - raise - finally: - await file.seek(0) - return path - - # ── 业务方法 ──────────────────────────────────────────────────── - - async def upload( - self, - source_id: int | None, - file: UploadFile, - remote_path: str | None = None, - ) -> dict: - """上传文件到远端存储。remote_path 为空时自动生成安全文件名。""" - if not file or not file.filename: - raise CustomException(msg="请选择要上传的文件") - - if not UploadUtil.check_path_traversal(file.filename): - raise CustomException(msg="文件名包含非法字符") - extension = UploadUtil.get_extension_from_filename(file.filename) - if not extension: - raise CustomException(msg="无法识别文件类型") - if UploadUtil.is_dangerous_extension(extension): - raise CustomException(msg=f"不允许上传此类型的文件: {extension}") - UploadUtil.check_file_size(file) - - # 确定远端路径 - if remote_path: - if remote_path.endswith("/"): - # 以 / 结尾视为目录：保留原文件名，拼接到目录下 - dir_path = self._validate_remote_path(remote_path) - target = f"{dir_path}/{file.filename}" - else: - target = self._validate_remote_path(remote_path) - if not target.endswith(extension): - target = f"{target}{extension}" - else: - target = UploadUtil.generate_safe_filename(file.filename, extension) - - config = await self._get_source(source_id) - temp_path = await self._save_to_temp(file, suffix=extension) - adapter = StorageAdapterFactory.create(config) - try: - await adapter.upload(temp_path, target) - file_url = await adapter.get_url(target) - finally: - await adapter.close() - os.unlink(temp_path) - - return { - "file_path": target, - "file_name": Path(target).name, - "origin_name": file.filename, - "file_url": file_url, - } - - async def download(self, source_id: int | None, remote_path: str) -> tuple[str, str]: - """下载远端文件到临时目录，返回 (本地临时路径, 文件名)。""" - target = self._validate_remote_path(remote_path) - config = await self._get_source(source_id) - extension = Path(target).suffix - fd, temp_path = tempfile.mkstemp(suffix=extension) - os.close(fd) - adapter = StorageAdapterFactory.create(config) - try: - local_path = await adapter.download(target, temp_path) - except Exception: - os.unlink(temp_path) - raise - finally: - await adapter.close() - return local_path, Path(target).name - - async def delete(self, source_id: int | None, remote_path: str) -> None: - target = self._validate_remote_path(remote_path) - config = await self._get_source(source_id) - adapter = StorageAdapterFactory.create(config) - try: - await adapter.delete(target) - finally: - await adapter.close() - - async def exists(self, source_id: int | None, remote_path: str) -> bool: - target = self._validate_remote_path(remote_path) - config = await self._get_source(source_id) - adapter = StorageAdapterFactory.create(config) - try: - return await adapter.exists(target) - finally: - await adapter.close() - - async def list(self, source_id: int | None, prefix: str = "") -> list[StorageObject]: - safe_prefix = self._validate_remote_path(prefix) if prefix else "" - config = await self._get_source(source_id) - adapter = StorageAdapterFactory.create(config) - try: - return await adapter.list(safe_prefix) - finally: - await adapter.close() - - async def get_url(self, source_id: int | None, remote_path: str, expire: int = 3600) -> str | None: - target = self._validate_remote_path(remote_path) - config = await self._get_source(source_id) - adapter = StorageAdapterFactory.create(config) - try: - return await adapter.get_url(target, expire=expire) - finally: - await adapter.close() - - async def copy_or_move( - self, - source_id: int | None, - source_path: str, - target_id: int, - target_path: str, - move: bool = False, - ) -> dict: - """复制/移动文件：跨端点时下载到临时再上传；同端点 move 即重命名。""" - src = self._validate_remote_path(source_path) - dst = self._validate_remote_path(target_path) - if move and source_id == target_id and src == dst: - raise CustomException(msg="源路径与目标路径相同") - source_config = await self._get_source(source_id) - target_config = await self._get_source(target_id) - fd, temp_path = tempfile.mkstemp(suffix=Path(dst).suffix) - os.close(fd) - src_adapter = StorageAdapterFactory.create(source_config) - dst_adapter = StorageAdapterFactory.create(target_config) - try: - await src_adapter.download(src, temp_path) - await dst_adapter.upload(temp_path, dst) - if move: - await src_adapter.delete(src) - finally: - await src_adapter.close() - await dst_adapter.close() - os.unlink(temp_path) - return {"source_path": src, "target_path": dst} diff --git a/backend/app/api/v1/module_storage/plugin.toml b/backend/app/api/v1/module_storage/plugin.toml deleted file mode 100644 index d4fc407d..00000000 --- a/backend/app/api/v1/module_storage/plugin.toml +++ /dev/null @@ -1,8 +0,0 @@ -# 见 docs/PLUGIN_ARCHITECTURE.md - -name = "storage" -title = "存储" -version = "1.0.0" -description = "多协议存储系统（FTP/FTPS/SFTP/S3/OBS/OSS/COS）" -optional = true -tags = ["storage", "ftp", "s3"] diff --git a/backend/app/api/v1/module_storage/transfer/engine.py b/backend/app/api/v1/module_storage/transfer/engine.py deleted file mode 100644 index b23ddd5d..00000000 --- a/backend/app/api/v1/module_storage/transfer/engine.py +++ /dev/null @@ -1,263 +0,0 @@ -"""文件传输任务执行引擎 - -- parallel（多目标）：单源依次输出到多个目标端点 -- chain（链式）：步骤串联，上一步目标端点即下一步源，链条长度不限 -- 进度按步骤粒度统计（SDK 无逐字节回调），实时写入 DB 并经 WebSocket 推送 -- 后台任务在独立 DB 会话中运行，不阻塞请求；取消采用内存标志（当前步骤执行完毕后生效） -""" - -import os -import tempfile -from datetime import UTC, datetime - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.api.v1.module_storage.core.base import StorageAdapterConfig -from app.api.v1.module_storage.core.constants import StorageProtocol -from app.api.v1.module_storage.core.encrypt import decrypt_password -from app.api.v1.module_storage.core.factory import StorageAdapterFactory -from app.api.v1.module_storage.source.model import StorageSourceModel -from app.api.v1.module_storage.transfer.registry import transfer_task_registry -from app.api.v1.module_storage.transfer.ws_manager import transfer_ws_manager -from app.core.database import async_db_session -from app.core.logger import logger - -from .model import StorageTransferStepModel, StorageTransferTaskModel - -# 单步最大可显示进度（步骤执行中为流动状态，完成后置 100） -_STEP_RUNNING_PROGRESS = 50 - - -def _dt(value: datetime | None) -> str | None: - return value.isoformat() if value else None - - -def _step_payload(step: StorageTransferStepModel) -> dict: - return { - "id": step.id, - "step_order": step.step_order, - "source_id": step.source_id, - "source_path": step.source_path, - "target_id": step.target_id, - "target_path": step.target_path, - "status": step.status, - "progress": step.progress, - "speed": step.speed, - "total_size": step.total_size, - "transferred_size": step.transferred_size, - "error_msg": step.error_msg, - "started_at": _dt(step.started_at), - "finished_at": _dt(step.finished_at), - } - - -def _task_payload(task: StorageTransferTaskModel, steps: list[StorageTransferStepModel]) -> dict: - return { - "id": task.id, - "name": task.name, - "task_type": task.task_type, - "source_type": task.source_type, - "source_id": task.source_id, - "source_path": task.source_path, - "source_name": task.source_name, - "source_size": task.source_size, - "status": task.status, - "total_size": task.total_size, - "transferred_size": task.transferred_size, - "progress": task.progress, - "speed": task.speed, - "error_msg": task.error_msg, - "started_at": _dt(task.started_at), - "finished_at": _dt(task.finished_at), - "steps": [_step_payload(s) for s in steps], - } - - -async def _broadcast(task: StorageTransferTaskModel, steps: list[StorageTransferStepModel]) -> None: - await transfer_ws_manager.send_to_user( - task.created_id, - {"type": "task_update", "data": _task_payload(task, steps)}, - ) - - -async def _build_config(db: AsyncSession, source_id: int) -> StorageAdapterConfig | None: - source = await db.get(StorageSourceModel, source_id) - if source is None or source.status == 1: - return None - return StorageAdapterConfig( - protocol=StorageProtocol(source.protocol), - host=source.host, - port=source.port, - username=source.username, - password=decrypt_password(source.password), - bucket=source.bucket, - endpoint=source.endpoint, - region=source.region, - path_prefix=source.path_prefix, - is_secure=source.is_secure, - implicit_tls=source.implicit_tls, - ) - - -async def _resolve_source_size(adapter, source_path: str) -> int: - """尽力获取远端源文件大小（对象存储/FTP 均可通过 list 精确匹配）。""" - try: - objects = await adapter.list(source_path) - for obj in objects: - if not obj.is_dir and obj.key == source_path and obj.size: - return obj.size - except Exception: - pass - return 0 - - -async def _run_step(db: AsyncSession, task: StorageTransferTaskModel, step: StorageTransferStepModel) -> bool: - """执行单个传输步骤，成功返回 True。""" - started_at = datetime.now(UTC) - step.status = "running" - step.started_at = started_at - step.progress = _STEP_RUNNING_PROGRESS - await db.commit() - await _broadcast(task, await _load_steps(db, task.id)) - - temp_path: str | None = None - src_adapter = None - dst_adapter = None - try: - # 解析源：本地临时文件直接使用；远端源下载到临时文件 - if step.source_id is not None: - src_config = await _build_config(db, step.source_id) - if src_config is None: - raise RuntimeError(f"源存储源 {step.source_id} 不存在或已停用") - src_adapter = StorageAdapterFactory.create(src_config) - fd, temp_path = tempfile.mkstemp(prefix="transfer_", suffix=os.path.splitext(step.target_path)[1]) - os.close(fd) - await src_adapter.download(step.source_path or "", temp_path) - else: - temp_path = step.source_path or "" - - if not temp_path or not os.path.exists(temp_path): - raise RuntimeError("源文件不存在") - - size = os.path.getsize(temp_path) - dst_config = await _build_config(db, step.target_id) - if dst_config is None: - raise RuntimeError(f"目标存储源 {step.target_id} 不存在或已停用") - dst_adapter = StorageAdapterFactory.create(dst_config) - await dst_adapter.upload(temp_path, step.target_path) - - elapsed = (datetime.now(UTC) - started_at).total_seconds() or 0.01 - speed = size / elapsed - step.total_size = size - step.transferred_size = size - step.speed = speed - step.status = "success" - step.progress = 100 - step.finished_at = datetime.now(UTC) - task.transferred_size += size - task.speed = speed - if task.total_size > 0: - task.progress = min(99, int(task.transferred_size * 100 / task.total_size)) - await db.commit() - await _broadcast(task, await _load_steps(db, task.id)) - return True - except Exception as e: - msg = str(e) or e.__class__.__name__ - step.status = "failed" - step.error_msg = msg - step.finished_at = datetime.now(UTC) - task.status = "failed" - task.error_msg = msg - task.finished_at = datetime.now(UTC) - await db.commit() - await _broadcast(task, await _load_steps(db, task.id)) - logger.warning("传输任务 {}(步骤 {}) 失败: {}", task.id, step.step_order, msg) - return False - finally: - if src_adapter is not None: - await src_adapter.close() - if dst_adapter is not None: - await dst_adapter.close() - if temp_path and step.source_id is not None and os.path.exists(temp_path): - os.unlink(temp_path) - - -async def _load_steps(db: AsyncSession, task_id: int) -> list[StorageTransferStepModel]: - result = await db.execute( - select(StorageTransferStepModel).where(StorageTransferStepModel.task_id == task_id).order_by(StorageTransferStepModel.step_order) - ) - return list(result.scalars().all()) - - -async def execute_transfer_task(task_id: int) -> None: - """后台执行传输任务（由创建接口以 asyncio.create_task 启动）。""" - async with async_db_session() as db: - task = await db.get(StorageTransferTaskModel, task_id) - if task is None or task.status != "pending": - return - steps = await _load_steps(db, task_id) - if not steps: - task.status = "failed" - task.error_msg = "任务没有可执行的步骤" - task.finished_at = datetime.now(UTC) - await db.commit() - return - - # 解析源文件大小，用于总进度估算 - if task.source_type == "local" and task.source_path and os.path.exists(task.source_path): - task.source_size = os.path.getsize(task.source_path) - elif task.source_type == "remote" and task.source_id: - config = await _build_config(db, task.source_id) - if config is None: - task.status = "failed" - task.error_msg = f"源存储源 {task.source_id} 不存在或已停用" - task.finished_at = datetime.now(UTC) - await db.commit() - await _broadcast(task, steps) - return - adapter = StorageAdapterFactory.create(config) - try: - task.source_size = await _resolve_source_size(adapter, task.source_path or "") - finally: - await adapter.close() - # 总字节 = 源大小 × 步骤数（每步传输一次源文件，parallel 与 chain 相同） - task.total_size = (task.source_size or 0) * len(steps) - task.status = "running" - task.started_at = datetime.now(UTC) - await db.commit() - await _broadcast(task, steps) - - completed = 0 - canceled = False - for step in steps: - if transfer_task_registry.is_canceled(task_id): - canceled = True - break - if await _run_step(db, task, step): - completed += 1 - else: - break - - if canceled: - task.status = "canceled" - task.error_msg = None - for step in steps: - if step.status == "pending": - step.status = "canceled" - step.finished_at = datetime.now(UTC) - elif completed == len(steps): - task.status = "success" - task.progress = 100 - task.finished_at = datetime.now(UTC) - transfer_task_registry.clear(task_id) - await db.commit() - await _broadcast(task, steps) - logger.info("传输任务 {} 结束: {}", task_id, task.status) - - # 清理本地源临时文件 - if task.source_type == "local" and task.source_path and os.path.exists(task.source_path): - try: - os.unlink(task.source_path) - except OSError: - pass diff --git a/backend/app/api/v1/module_storage/transfer/service.py b/backend/app/api/v1/module_storage/transfer/service.py deleted file mode 100644 index 7ddfef04..00000000 --- a/backend/app/api/v1/module_storage/transfer/service.py +++ /dev/null @@ -1,181 +0,0 @@ -import asyncio -import os -import tempfile -from datetime import UTC, datetime - -import aiofiles -from fastapi import UploadFile -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.api.v1.module_storage.source.service import StorageSourceService -from app.api.v1.module_storage.transfer.engine import execute_transfer_task -from app.api.v1.module_storage.transfer.registry import transfer_task_registry -from app.core.base_schema import AuthSchema, PageResultSchema -from app.core.exceptions import CustomException -from app.utils.common_util import search_to_dict - -from .crud import StorageTransferTaskCRUD -from .model import StorageTransferStepModel -from .schema import TransferStepOutSchema, TransferTargetSchema, TransferTaskCreateSchema, TransferTaskOutSchema, TransferTaskQueryParam - - -class StorageTransferService: - """文件传输任务服务（创建 / 查询 / 取消 / 删除）""" - - def __init__(self, auth: AuthSchema, db: AsyncSession) -> None: - self.auth = auth - self.db = db - - # ── 内部工具 ──────────────────────────────────────────────────── - - def _crud(self) -> StorageTransferTaskCRUD: - return StorageTransferTaskCRUD(self.auth, self.db) - - async def _validate_targets(self, targets: list[TransferTargetSchema]) -> None: - """校验目标存储源均存在且启用。""" - source_service = StorageSourceService(self.auth, self.db) - for target in targets: - await source_service.get_active_source(target.target_id) - - @staticmethod - def _build_steps(data: TransferTaskCreateSchema, local_source_path: str | None = None) -> list[dict]: - """展开步骤：chain 下每步源继承上一步的目标；parallel 下每步源均为任务源。 - - local 源时任务源为服务端临时文件，需显式传入 local_source_path。 - """ - steps: list[dict] = [] - if data.task_type == "chain": - prev_id, prev_path = data.source_id, local_source_path or data.source_path - for order, target in enumerate(data.targets): - steps.append( - { - "step_order": order, - "source_id": prev_id, - "source_path": prev_path, - "target_id": target.target_id, - "target_path": target.target_path, - } - ) - prev_id, prev_path = target.target_id, target.target_path - else: - for order, target in enumerate(data.targets): - steps.append( - { - "step_order": order, - "source_id": data.source_id, - "source_path": local_source_path or data.source_path, - "target_id": target.target_id, - "target_path": target.target_path, - } - ) - return steps - - async def _persist(self, data: TransferTaskCreateSchema, local_info: dict | None = None) -> int: - """落库任务与步骤（pending），随后启动后台执行。""" - task = await self._crud().create( - { - "name": data.name, - "task_type": data.task_type, - "source_type": data.source_type, - "source_id": data.source_id, - "source_path": local_info["source_path"] if local_info else data.source_path, - "source_name": local_info["source_name"] if local_info else ((data.source_path or "").rsplit("/", 1)[-1] or None), - "source_size": local_info.get("source_size") if local_info else None, - "status": "pending", - } - ) - local_source_path = local_info["source_path"] if local_info else None - for step_data in self._build_steps(data, local_source_path=local_source_path): - self.db.add(StorageTransferStepModel(task_id=task.id, **step_data)) - await self.db.commit() - await self._launch(task.id) - return task.id - - async def _launch(self, task_id: int) -> None: - """启动后台执行任务（不阻塞请求）。""" - asyncio.create_task(execute_transfer_task(task_id)) - - # ── 创建 ──────────────────────────────────────────────────────── - - async def create(self, data: TransferTaskCreateSchema) -> int: - """创建远端源传输任务。""" - source_service = StorageSourceService(self.auth, self.db) - if data.source_type == "remote": - await source_service.get_active_source(data.source_id) - await self._validate_targets(data.targets) - return await self._persist(data) - - async def create_local(self, data: TransferTaskCreateSchema, file: UploadFile) -> int: - """创建本地源传输任务：文件保存到服务端临时目录，执行完毕后自动清理。""" - if not file or not file.filename: - raise CustomException(msg="请选择要上传的文件") - await self._validate_targets(data.targets) - fd, temp_path = tempfile.mkstemp(prefix="transfer_upload_", suffix=os.path.splitext(file.filename)[1]) - os.close(fd) - try: - async with aiofiles.open(temp_path, "wb") as f: - while chunk := await file.read(1024 * 1024): - await f.write(chunk) - except Exception: - os.unlink(temp_path) - raise - finally: - await file.seek(0) - return await self._persist( - data, - local_info={"source_path": temp_path, "source_name": file.filename, "source_size": os.path.getsize(temp_path)}, - ) - - # ── 查询 ──────────────────────────────────────────────────────── - - async def page( - self, - search: TransferTaskQueryParam | None, - page_no: int, - page_size: int, - order_by: list[dict] | None = None, - ) -> PageResultSchema[TransferTaskOutSchema]: - result = await self._crud().page( - offset=(page_no - 1) * page_size, - limit=page_size, - order_by=order_by or [{"id": "desc"}], - search=search_to_dict(search), - ) - return PageResultSchema[TransferTaskOutSchema]( - page_no=result.page_no, - page_size=result.page_size, - total=result.total, - has_next=result.has_next, - items=[TransferTaskOutSchema.model_validate(obj) for obj in result.items], - ) - - async def detail(self, task_id: int) -> TransferTaskOutSchema: - task = await self._crud().get_or_404(id=task_id) - out = TransferTaskOutSchema.model_validate(task) - result = await self.db.execute( - select(StorageTransferStepModel) - .where( - StorageTransferStepModel.task_id == task_id, - StorageTransferStepModel.is_deleted.is_(False), - ) - .order_by(StorageTransferStepModel.step_order) - ) - out.steps = [TransferStepOutSchema.model_validate(step) for step in result.scalars().all()] - return out - - # ── 操作 ──────────────────────────────────────────────────────── - - async def cancel(self, task_id: int) -> None: - task = await self._crud().get_or_404(id=task_id) - if task.status == "pending": - task.status = "canceled" - task.finished_at = datetime.now(UTC) - await self.db.flush() - elif task.status == "running": - transfer_task_registry.mark_cancel(task_id) - - async def delete(self, ids: list[int]) -> None: - for task_id in ids: - transfer_task_registry.mark_cancel(task_id) - await self._crud().delete(ids=ids) diff --git a/backend/app/api/v1/module_storage/transfer/ws_manager.py b/backend/app/api/v1/module_storage/transfer/ws_manager.py deleted file mode 100644 index 13e840a7..00000000 --- a/backend/app/api/v1/module_storage/transfer/ws_manager.py +++ /dev/null @@ -1,34 +0,0 @@ -"""传输任务 WebSocket 连接管理（单实例部署，按用户推送任务进度）""" - -from fastapi import WebSocket - - -class TransferWSManager: - """维护 user_id -> 连接集合，支持多标签页连接""" - - def __init__(self) -> None: - self._connections: dict[int, set[WebSocket]] = {} - - async def connect(self, user_id: int, ws: WebSocket) -> None: - await ws.accept() - self._connections.setdefault(user_id, set()).add(ws) - - def disconnect(self, user_id: int, ws: WebSocket) -> None: - conns = self._connections.get(user_id) - if conns is None: - return - conns.discard(ws) - if not conns: - self._connections.pop(user_id, None) - - async def send_to_user(self, user_id: int | None, data: dict) -> None: - if user_id is None: - return - for ws in list(self._connections.get(user_id, ())): - try: - await ws.send_json(data) - except Exception: - pass - - -transfer_ws_manager = TransferWSManager() diff --git a/backend/app/api/v1/module_system/auth/controller.py b/backend/app/api/v1/module_system/auth/controller.py index 671801af..25385032 100644 --- a/backend/app/api/v1/module_system/auth/controller.py +++ b/backend/app/api/v1/module_system/auth/controller.py @@ -1,24 +1,9 @@ -""" -认证控制器 — TODO: 限流粒度细化 ---------------------------------- -当前登录（/login）和 OAuth 端点（/oauth/*）共享应用的通用限流配置， -缺少独立的、更严格的限流策略。建议为以下端点配置独立的 RateLimiter： +"""认证控制器。 -1. /auth/login — 密码登录 - - 建议: 按 IP + 用户名组合限流，如 5次/分钟/IP + 10次/15分钟/用户 - - 原因: 暴力破解防护 - -2. /auth/oauth/* — 第三方 OAuth 登录/回调 - - 建议: 按 IP 限流，如 10次/分钟/IP - - 原因: OAuth 流程可能触发多次重定向，频率稍高于登录 - -3. /auth/captcha/* — 验证码获取/校验 - - 建议: 按 IP 限流，如 3次/分钟/IP - - 原因: 防止验证码遍历 +TODO(限流细化): /login、/oauth/*、/captcha/* 目前共享全局限流，建议为各端点配置 +独立的 RateLimiter——登录按 IP+用户名限流防爆破、OAuth 与验证码按 IP 限流防遍历。 """ -import json -import secrets from typing import Annotated from fastapi import APIRouter, BackgroundTasks, Body, Depends, Path, Query, Request @@ -26,29 +11,15 @@ from fastapi.responses import JSONResponse, RedirectResponse from redis.asyncio.client import Redis from sqlalchemy.ext.asyncio import AsyncSession -from app.api.v1.module_system.user.crud import UserCRUD -from app.api.v1.module_system.user.schema import UserCreateSchema -from app.api.v1.module_system.user.service import UserService from app.common.response import ErrorResponse, RedirectContentResponse, ResponseSchema, SuccessResponse from app.config.setting import settings -from app.core.base_schema import AuthSchema, JWTOutSchema +from app.core.base_schema import JWTOutSchema from app.core.dependencies import db_getter, get_current_user, redis_getter -from app.core.exceptions import CustomException from app.core.logger import logger -from app.core.redis_crud import RedisCURD from app.core.router_class import OperationLogRoute from app.core.security import CustomOAuth2PasswordRequestForm -from .oauth_service import ( - STATE_PREFIX, - OAuthProvider, - _callback_url, - build_authorize_url, - complete_oauth_login, - oauth_service_error_redirect, - oauth_service_frontend_redirect_from_token, - save_oauth_state, -) +from .oauth_service import OAuthProvider, finish_oauth_login, start_oauth_login from .schema import ( CaptchaOutSchema, LoginOutSchema, @@ -63,12 +34,7 @@ from .service import ( CaptchaService, LoginService, ) -from .wx_mini_service import ( - code2session, - ensure_wx_user, - get_phone_number, - get_qrcode, -) +from .wx_mini_service import get_qrcode, wx_mini_login, wx_mini_phone_login AuthRouter = APIRouter(route_class=OperationLogRoute, prefix="/auth", tags=["认证授权"]) @@ -102,18 +68,20 @@ async def get_new_token_controller( @AuthRouter.get("/captcha/get", summary="获取验证码", response_model=ResponseSchema[CaptchaOutSchema]) async def get_captcha_for_login_controller( + request: Request, redis: Annotated[Redis, Depends(redis_getter)], ) -> JSONResponse: - captcha = await CaptchaService.get_captcha(redis=redis) + captcha = await CaptchaService.get_captcha(redis=redis, request=request) return SuccessResponse(data=captcha, msg="获取验证码成功") @AuthRouter.post("/captcha/slider/complete", summary="滑块验证完成", response_model=ResponseSchema[SliderCompleteOutSchema]) async def slider_complete_controller( + request: Request, redis: Annotated[Redis, Depends(redis_getter)], body: SliderCompleteSchema, ) -> JSONResponse: - result = await CaptchaService.slider_complete(redis=redis, captcha_key=body.captcha_key) + result = await CaptchaService.slider_complete(redis=redis, request=request, captcha_key=body.captcha_key) return SuccessResponse(data=result, msg="滑块验证成功") @@ -135,34 +103,14 @@ async def oauth_login_redirect_controller( provider: Annotated[OAuthProvider, Path(description="wechat | qq | github | gitee")], redirect_uri: Annotated[str | None, Query(description="OAuth 完成后浏览器回到的前端登录页完整 URL")] = None, ) -> RedirectResponse: - allowed = {"wechat", "qq", "github", "gitee"} - fe = redirect_uri or settings.OAUTH_FRONTEND_FALLBACK - if provider not in allowed: - return RedirectContentResponse( - url=oauth_service_error_redirect(fe, "不支持的 OAuth 渠道"), - status_code=302, - ) - if not redirect_uri: - return RedirectContentResponse( - url=oauth_service_error_redirect(fe, "缺少 redirect_uri 参数"), - status_code=302, - ) - try: - state = secrets.token_urlsafe(32) - await save_oauth_state( - redis=redis, - state=state, - provider=provider, - frontend_redirect=redirect_uri, - ) - cb = _callback_url(request, provider) - url = build_authorize_url(provider=provider, callback_url=cb, state=state) - return RedirectContentResponse(url=url, status_code=302) - except CustomException as e: - return RedirectContentResponse( - url=oauth_service_error_redirect(redirect_uri, e.msg), - status_code=302, - ) + """跳转第三方授权页；入参缺失或渠道密钥未配置时自动降级为错误重定向。""" + url = await start_oauth_login( + request=request, + redis=redis, + provider=provider, + redirect_uri=redirect_uri, + ) + return RedirectContentResponse(url=url, status_code=302) @AuthRouter.get("/oauth/{provider}/callback", summary="第三方OAuth回调", include_in_schema=False) @@ -175,43 +123,17 @@ async def oauth_callback_controller( code: Annotated[str | None, Query(description="OAuth 授权码")] = None, state: Annotated[str | None, Query(description="OAuth 状态参数")] = None, ) -> RedirectResponse: - fe_fallback = settings.OAUTH_FRONTEND_FALLBACK - - async def resolve_frontend() -> str: - if not state: - return fe_fallback - raw = await RedisCURD(redis).get(f"{STATE_PREFIX}{state}") - if not raw: - return fe_fallback - if isinstance(raw, bytes): - raw = raw.decode("utf-8") - try: - payload = json.loads(raw) - return str(payload.get("frontend_redirect") or fe_fallback).strip() or fe_fallback - except json.JSONDecodeError: - return fe_fallback - - if provider not in {"wechat", "qq", "github", "gitee"}: - url = oauth_service_error_redirect(await resolve_frontend(), "不支持的 OAuth 渠道") - return RedirectContentResponse(url=url, status_code=302) - if not code or not state: - url = oauth_service_error_redirect(await resolve_frontend(), "授权被取消或参数不完整") - return RedirectContentResponse(url=url, status_code=302) - try: - token, fe = await complete_oauth_login( - request=request, - redis=redis, - db=db, - provider=provider, - code=code, - state=state, - background_tasks=background_tasks, - ) - success_url = oauth_service_frontend_redirect_from_token(fe, token) - return RedirectContentResponse(url=success_url, status_code=302) - except CustomException as e: - fe = await resolve_frontend() - return RedirectContentResponse(url=oauth_service_error_redirect(fe, e.msg), status_code=302) + """处理第三方平台回调并回跳前端登录页（成功携带令牌 / 失败携带原因）。""" + url = await finish_oauth_login( + request=request, + redis=redis, + db=db, + provider=provider, + code=code, + state=state, + background_tasks=background_tasks, + ) + return RedirectContentResponse(url=url, status_code=302) # =================================================== # @@ -227,56 +149,17 @@ async def wx_mini_login_controller( body: WxLoginSchema, background_tasks: BackgroundTasks, ) -> JSONResponse: - """微信小程序登录（code2Session）。 - - 前端通过 uni.login 获取 code，后端调用微信 code2Session 接口换取 openid， - 然后查找或自动注册用户，最终返回 JWT。 - """ - session_data = await code2session(code=body.code) - openid = session_data["openid"] - - user = await ensure_wx_user( - db=db, - openid=openid, - nickname=body.nickname, - avatar=body.avatar, - ) - - if user.status == 1: - raise CustomException(msg="用户已被停用") - - user = await UserCRUD(AuthSchema(), db).update_last_login(id=user.id) - if not user: - raise CustomException(msg="用户不存在") - - token = await LoginService.create_token( + """微信小程序登录：前端 uni.login 的 code 换取 openid 后签发 JWT。""" + result = await wx_mini_login( request=request, redis=redis, - user=user, - login_type="wx_mini", + db=db, + code=body.code, + nickname=body.nickname, + avatar=body.avatar, background_tasks=background_tasks, ) - - user_info = { - "id": user.id, - "username": user.username, - "name": user.name, - "avatar": user.avatar, - "is_superuser": user.is_superuser, - } - - logger.info(f"微信小程序用户登录成功: {user.username}") - - return SuccessResponse( - data=LoginOutSchema( - access_token=token.access_token, - refresh_token=token.refresh_token, - expires_in=token.expires_in, - token_type=token.token_type, - user_info=user_info, - ), - msg="登录成功", - ) + return SuccessResponse(data=result, msg="登录成功") @AuthRouter.post("/wx-phone-login", summary="微信小程序手机号登录", response_model=ResponseSchema[LoginOutSchema]) @@ -287,82 +170,15 @@ async def wx_mini_phone_login_controller( body: WxPhoneLoginSchema, background_tasks: BackgroundTasks, ) -> JSONResponse: - """微信小程序手机号快速登录。 - - 前端 + + + + + {{ currentTime }} diff --git a/frontend/web/src/views/fastlink/current/profile.vue b/frontend/web/src/views/fastlink/current/profile.vue index 9fce09d9..63f1708e 100644 --- a/frontend/web/src/views/fastlink/current/profile.vue +++ b/frontend/web/src/views/fastlink/current/profile.vue @@ -502,10 +502,11 @@ async function onAvatarCropConfirm(dataURL: string) { } } -function normalizeGenderValue(v: string | number | undefined): number { - if (v === undefined || v === null || v === "") return 1; +/** 性别统一为字符串，与后端 CurrentUserUpdateSchema(gender: str) 对齐 */ +function normalizeGenderValue(v: string | number | undefined): string { + if (v === undefined || v === null || v === "") return "1"; const n = typeof v === "string" ? Number(v) : v; - return Number.isFinite(n) ? n : 1; + return Number.isFinite(n) ? String(n) : "1"; } const initInfoForm = () => { @@ -705,7 +706,15 @@ const handleSave = async () => { if (!valid) { return false; } - const response = await UserAPI.updateCurrentUserInfo({ ...infoFormState }); + // 只提交后端可编辑字段，避免携带 dept/roles/positions 等冗余信息 + const response = await UserAPI.updateCurrentUserInfo({ + name: infoFormState.name, + gender: infoFormState.gender, + mobile: infoFormState.mobile, + email: infoFormState.email, + avatar: infoFormState.avatar, + description: infoFormState.description, + }); await userStore.setUserInfo(response.data.data); initInfoForm(); ElMessage.success("个人资料已保存"); diff --git a/frontend/web/src/views/fastlink/fachat/index.vue b/frontend/web/src/views/fastlink/fachat/index.vue index c153d672..e287c303 100644 --- a/frontend/web/src/views/fastlink/fachat/index.vue +++ b/frontend/web/src/views/fastlink/fachat/index.vue @@ -3,150 +3,161 @@
-
-
-
- -
-
{{ selectedPerson?.name }}
-
{{ selectedPerson?.email }}
-
-
-
- -
- - - 排序方式 - - - - - - -
- -
-
- - - {{ item.name.charAt(0) }} - - -
-
-
- {{ item.name }} - {{ item.lastTime }} +
+
+
+ +
+
{{ selectedPerson?.name }}
+
{{ selectedPerson?.email }}
+
-
- - {{ item.email }} +
+ +
+ + + 排序方式 + + + -
+ +
-
- + +
+
+ + + {{ item.name.charAt(0) }} + + +
+
+
+ {{ item.name }} + {{ item.lastTime }} +
+
+ + {{ item.email }} + +
+
+
+
-
-
- Art Bot -
-
- {{ isOnline ? "在线" : "离线" }} +
+
+ Art Bot +
+
+ {{ isOnline ? "在线" : "离线" }} +
+
+
+ + + +
-
-
- - - -
-
-
- - - + - -
- - - -
-
- -
- 发送
-
-
diff --git a/frontend/web/src/views/fastlink/tutorial/index.vue b/frontend/web/src/views/fastlink/tutorial/index.vue index f96ae276..fd1cac29 100644 --- a/frontend/web/src/views/fastlink/tutorial/index.vue +++ b/frontend/web/src/views/fastlink/tutorial/index.vue @@ -90,727 +90,745 @@ class="manual-feature-body__scrollbar fa-card-sm rounded-custom-sm h-full" max-height="min(78vh, 880px)" > - -
-
-

- FastapiAdmin 功能点清单 - 用于全功能测试验收，按模块逐页列出所有可操作元素 -

+ +
+
+

+ FastapiAdmin 功能点清单 + 用于全功能测试验收，按模块逐页列出所有可操作元素 +

- - - -
    -
  • - - 一、系统管理 - -
    - - 用户管理 + + + +
      +
    • + + 一、系统管理 - · - - 角色管理 +
      + + 用户管理 + + · + + 角色管理 + + · + + 菜单管理 + + · + + 部门管理 + + · + + 岗位管理 + + · + + 字典管理 + + · + + 参数配置 + + · + + 通知公告 + + · + + 操作日志 + + · + + 登录页 + +
      +
      • +
    • + + 二、监控管理 - · - - 菜单管理 +
      + + 在线用户 + + · + + 缓存管理 + + · + + 文件管理 + + · + + 服务监控 + +
      +
      • +
    • + + 三、任务管理 - · - - 部门管理 +
      + + 调度器监控 + + · + + 节点管理 + + · + + 流程编排 + + · + + 节点类型 + +
      +
      • +
    • + + 四、AI 模块 - · - - 岗位管理 +
      + + AI智能助手 + + · + + 会话聊天 + + · + + 会话记忆 + +
      +
      • +
    • + + 五、代码生成器 - · - - 字典管理 +
      + + 代码生成 + +
      +
      • +
    • + + 六、应用管理 - · - - 参数配置 +
      + + 插件市场 + +
      +
      • +
    • + + 七、示例模块 - · - - 通知公告 +
      + + 示例管理 + +
      +
      • +
    • + + 八、仪表盘 - · - - 操作日志 +
      + 首页 + · + + 个人中心 + + · + + 更新日志 + + · + + 工作台 + + · + + 控制台 + + · + + 分析页 + + · + + 电子商务 + + · + + 地图 + + · + + 定价 + + · + + 文章管理 + + · + + 教程 + +
      +
      • +
    • + + 九、布局与通用功能 - · - - 登录页 +
      + + 主布局 + + · + + 侧栏菜单 + + · + + 顶栏 + + · + + 标签页 + + · + + 设置面板 + + · + + 通知 + + · + + 全局搜索 + + · + + 锁屏 + + · + + 用户菜单 + + · + + 主题切换 + + · + + 语言切换 + +
      +
      • +
    • + + 十、异常页 +
      + 401 + · + 403 + · + 404 + · + 500 +
      +
      • +
    • + + 十一、接口文档（API） + +
      + + Swagger文档 + + · + + Redoc文档 + + · + + LangJin文档 + +
      +
      • +
    +     + + +
    +

    + 一、系统管理 + + module_system + +

    + + +
    +

    + 1.1 用户管理 + + module_system/user/index.vue + +

    + +
    + API 权限标识 +

    + + module_system:user + + — 含 + + create + + + delete + + + update + + + detail + + + import + + + export + + + patch + +

    -
    • -
  • - - 二、监控管理 - -
    - - 在线用户 - - · - - 缓存管理 - - · - - 文件管理 - - · - - 服务监控 - + +
    + + 🔍 搜索/筛选表单（5字段） + + + + + + + + + + + + + + + + + + + + + + + + + +
    备注
    文本·账号
    文本·用户名
    下拉·启用/停用
    创建人（FaUserTableSelect 弹窗选用户）
    创建时间·日期时间范围
    +
    -
    • -
  • - - 三、任务管理 - -
    - - 调度器监控 - - · - - 节点管理 - - · - - 流程编排 - - · - - 节点类型 - + +
    + 📊 表格列 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    渲染
    固定左侧
    ElAvatar
    溢出省略
    溢出省略
    + 启用 + 停用 +
    row.dept?.name
    + 男 + 女 + 未知 +
    固定右侧
    +
    -
    • -
  • - - 四、AI 模块 - -
    - - AI智能助手 - - · - - 会话聊天 - - · - - 会话记忆 - + +
    + 🔘 工具栏按钮 + + + 新增 + + + 导入 + + + 导出 + + + 删除 + + + 更多(批量启/停用) + + 刷新 + 列配置 +
    -
    • -
  • - - 五、代码生成器 - -
    - - 代码生成 - + +
    + 🔘 行操作按钮 + + + 重置密码 + + + 详情 + + + 编辑 + + + 删除 + +
    -
    • -
  • - - 六、应用管理 - -
    - - 插件市场 - + +
    + 📋 弹窗/抽屉 +
      +
    • + 详情 Drawer + — + 编号、头像、账号、用户名、性别(标签)、部门、角色(逗号拼接)、岗位(逗号拼接)、邮箱、手机号、是否超管(标签)、状态(标签)、上次登录时间、创建人、更新人、创建时间、更新时间、描述 +
      • +
    • + 新增/编辑 Drawer + (450px) — + 账号(username,编辑时禁用)、用户名(name)、性别、手机号(正则校验)、邮箱(正则校验)、部门(ElTreeSelect)、角色(多选)、岗位(多选)、密码(仅新增)、是否超管(Switch)、状态(Radio)、描述(textarea) +
      • +
    • + 导入弹窗 + — FaImportDialog, 模板 user_import_template.xlsx +
      • +
    • + 导出弹窗 + — FaExportDialog +
      • +
    • + 重置密码弹窗 + — 输入新密码, 至少6位 +
      • +
    -
    • -
  • - - 七、示例模块 - -
    - - 示例管理 - + +
    + ✨ 特殊功能 +
      +
    • 左侧部门树联动筛选(点击树节点过滤列表)
      • +
    • 批量删除(确认对话框)
      • +
    • 批量启用/停用
      • +
    • 若删除自己则清除登录信息登出
      • +
    -
    • -
  • - - 八、仪表盘 - -
    - 首页 - · - - 个人中心 - - · - - 更新日志 - - · - - 工作台 - - · - - 控制台 - - · - - 分析页 - - · - - 电子商务 - - · - - 地图 - - · - - 定价 - - · - - 文章管理 - - · - - 教程 - +
    + + +
    +

    + 1.2 角色管理 + + module_system/role/index.vue + +

    + +
    + API 权限标识 +

    + + module_system:role + + — 含 + + create + + + delete + + + update + + + detail + + + export + + + patch + + + permission + +

    -
    • -
  • - - 九、布局与通用功能 - -
    - - 主布局 - - · - - 侧栏菜单 - - · - - 顶栏 - - · - - 标签页 - - · - - 设置面板 - - · - - 通知 - - · - - 全局搜索 - - · - - 锁屏 - - · - - 用户菜单 - - · - - 主题切换 - - · - - 语言切换 - + +
    + + 🔍 搜索表单（3字段） + + + + + + + + + + + + + + + + + + + +
    类型
    文本输入
    下拉(启用/停用, value="true"/"false")
    日期时间范围
    +
    -
    • -
  • - - 十、异常页 - -
    - 401 - · - 403 - · - 404 - · - 500 + +
    + 📊 表格列 + + + + + + + + + + + + + + + + + + + + + + + + +
    渲染
    固定左侧
    溢出省略
    + 启用 + 停用 +
    溢出省略
    固定右侧
    +
    -
    • -
  • - - 十一、接口文档（API） - -
    - - Swagger文档 - - · - - Redoc文档 - - · - - LangJin文档 - + +
    + 🔘 工具栏按钮 + + + 新增 + + + 导出 + + + 删除 + + 刷新 + 列配置 +
    -
    • -
- - -
-

- 一、系统管理 - - module_system - -

+
+ 🔘 行操作按钮 + + + 权限 + + + 编辑 + + + 删除 + + +
- -
-

- 1.1 用户管理 - - module_system/user/index.vue - -

- -
- API 权限标识 -

- - module_system:user - - — 含 - - create - - - delete - - - update - - - detail - - - import - - - export - - - patch - -

+
+ 📋 弹窗/抽屉 +
    +
  • + 新增/编辑 Drawer + (450px) — 名称、标识(编辑时禁用)、排序、状态(Radio)、权限树(ElTree, + 勾选)、备注(textarea) +
    • +
  • + 权限 Drawer + (600px) — 权限菜单树(ElTree, 勾选, 支持展开/收起) +
    • +
  • + 导出弹窗 + — FaExportDialog +
    • +
+
-
- - 🔍 搜索/筛选表单（5字段） - - - - - - - - - - - - - - - - - - - - - - - - - -
备注
文本·账号
文本·用户名
下拉·启用/停用
创建人（FaUserTableSelect 弹窗选用户）
创建时间·日期时间范围
- -
- -
- 📊 表格列 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
渲染
固定左侧
ElAvatar
溢出省略
溢出省略
- 启用 - 停用 -
row.dept?.name
- 男 - 女 - 未知 -
固定右侧
- -
- -
- 🔘 工具栏按钮 - - - 新增 - - - 导入 - - - 导出 - - - 删除 - - - 更多(批量启/停用) - - 刷新 - 列配置 - -
- -
- 🔘 行操作按钮 - - - 重置密码 - - - 详情 - - - 编辑 - - - 删除 - - -
- -
- 📋 弹窗/抽屉 -
    -
  • - 详情 Drawer - — - 编号、头像、账号、用户名、性别(标签)、部门、角色(逗号拼接)、岗位(逗号拼接)、邮箱、手机号、是否超管(标签)、状态(标签)、上次登录时间、创建人、更新人、创建时间、更新时间、描述 -
    • -
  • - 新增/编辑 Drawer - (450px) — - 账号(username,编辑时禁用)、用户名(name)、性别、手机号(正则校验)、邮箱(正则校验)、部门(ElTreeSelect)、角色(多选)、岗位(多选)、密码(仅新增)、是否超管(Switch)、状态(Radio)、描述(textarea) -
    • -
  • - 导入弹窗 - — FaImportDialog, 模板 user_import_template.xlsx -
    • -
  • - 导出弹窗 - — FaExportDialog -
    • -
  • - 重置密码弹窗 - — 输入新密码, 至少6位 -
    • -
-
- -
- ✨ 特殊功能 -
    -
  • 左侧部门树联动筛选(点击树节点过滤列表)
    • -
  • 批量删除(确认对话框)
    • -
  • 批量启用/停用
    • -
  • 若删除自己则清除登录信息登出
    • -
-
-
- - -
-

- 1.2 角色管理 - - module_system/role/index.vue - -

- -
- API 权限标识 -

- - module_system:role - - — 含 - - create - - - delete - - - update - - - detail - - - export - - - patch - - - permission - -

-
- -
- - 🔍 搜索表单（3字段） - - - - - - - - - - - - - - - - - - - -
类型
文本输入
下拉(启用/停用, value="true"/"false")
日期时间范围
- -
- -
- 📊 表格列 - - - - - - - - - - - - - - - - - - - - - - - - -
渲染
固定左侧
溢出省略
- 启用 - 停用 -
溢出省略
固定右侧
- -
- -
- 🔘 工具栏按钮 - - - 新增 - - - 导出 - - - 删除 - - 刷新 - 列配置 - -
- -
- 🔘 行操作按钮 - - - 权限 - - - 编辑 - - - 删除 - - -
- -
- 📋 弹窗/抽屉 -
    -
  • - 新增/编辑 Drawer - (450px) — 名称、标识(编辑时禁用)、排序、状态(Radio)、权限树(ElTree, - 勾选)、备注(textarea) -
    • -
  • - 权限 Drawer - (600px) — 权限菜单树(ElTree, 勾选, 支持展开/收起) -
    • -
  • - 导出弹窗 - — FaExportDialog -
    • -
-
-
- - -
-

- {{ p.title }} - - {{ p.path }} - -

-
- - 功能完整性验收 - -
    -
  • {{ line }}
    • -
-
-
-
- - -
-

- {{ mod.heading }} - +
- {{ mod.pkgTag }} - -

-
-

- {{ p.title }} - - {{ p.path }} - -

-
- - 功能完整性验收 - -
    -
  • {{ line }}
    • -
+

+ {{ p.title }} + + {{ p.path }} + +

+
+ + 功能完整性验收 + +
    +
  • {{ line }}
    • +
+
-
-
-

- FastapiAdmin 功能点清单（完整性）— - 与当前代码中已实现界面项对齐，用于逐项核对是否漏测，不评价体验优劣。 -

+ +
+

+ {{ mod.heading }} + + {{ mod.pkgTag }} + +

+
+

+ {{ p.title }} + + {{ p.path }} + +

+
+ + 功能完整性验收 + +
    +
  • {{ line }}
    • +
+
+
+
+ +
+

+ FastapiAdmin 功能点清单（完整性）— + 与当前代码中已实现界面项对齐，用于逐项核对是否漏测，不评价体验优劣。 +

+
-
- +
diff --git a/frontend/web/src/views/fastlink/tutorial/manualSections.ts b/frontend/web/src/views/fastlink/tutorial/manualSections.ts index a3335f59..32710d5b 100644 --- a/frontend/web/src/views/fastlink/tutorial/manualSections.ts +++ b/frontend/web/src/views/fastlink/tutorial/manualSections.ts @@ -143,16 +143,6 @@ export const MANUAL_MODULES_AFTER_SYSTEM: ManualModuleSection[] = [ "完整性：两 Tab 均需点开核对是否加载成功、清理是否有二次确认。", ], }, - { - anchor: "page-resource", - title: "文件管理", - path: "module_monitor/resource/index.vue", - notes: [ - "检索：FaSearchBar；路径面包屑导航进入子目录。", - "工具栏：上传、新建文件夹、下载、删除等（`module_monitor:resource:*` 按钮权限）；刷新、列配置。", - "主区：文件表格；与后端资源浏览一致（菜单名「文件管理」，component `resource`）。", - ], - }, { anchor: "page-server", title: "服务监控", @@ -192,24 +182,44 @@ export const MANUAL_MODULES_AFTER_SYSTEM: ManualModuleSection[] = [ }, { anchor: "page-workflow", - title: "流程编排", + title: "传输流程", path: "module_task/workflow/flow/index.vue", notes: [ "检索：FaSearchBar（可展开）。", "工具栏：新增、批量删除、刷新、列配置。", - "行操作：草稿「发布」；已发布「执行」下拉；「编辑」打开 `WorkflowDesignDrawer` 画布；删除。", - "权限：`module_task:workflow:flow:create|delete|update|execute` 等（见行内 v-hasPerm）。", + "流程定义：源节点 + 目标节点列表（parallel 多目标 / chain 链式），目标路径随行配置。", + "权限：`module_task:workflow:flow:create|delete|update` + 行级操作。", ], }, { - anchor: "page-nodetype", - title: "节点类型", - path: "module_task/workflow/nodes/index.vue", + anchor: "page-workflow-node", + title: "节点管理", + path: "module_task/workflow/node/index.vue", notes: [ "检索：FaSearchBar（可展开）。", - "工具栏：新增、批量删除、刷新、列配置。", - "行操作：编辑（打开表单/脚本配置）、删除等（见 `nodes-operation` 槽）。", - "权限：`module_task:workflow:nodes:create|delete|update` + 行级操作。", + "节点即传输地址（存储源）：FTP/FTPS/SFTP/S3/OBS/OSS/COS/本地目录。", + "行操作：打开（进入存储管理）、测试连接、详情、编辑、删除。", + "权限：`module_task:workflow:node:create|delete|update` + 行级操作。", + ], + }, + { + anchor: "page-workflow-storage", + title: "存储管理", + path: "module_task/workflow/storage/index.vue", + notes: [ + "多节点标签浏览各节点内部文件（类似 IDE 标签栏）。", + "文件操作：上传、新建目录、下载、复制/移动、重命名、分享、删除。", + "可从「节点管理」点击「打开」直接跳入对应节点。", + ], + }, + { + anchor: "page-workflow-transfer", + title: "传输任务", + path: "module_task/workflow/transfer/index.vue", + notes: [ + "创建传输任务：可选「从流程自动填充」源节点与多目标，或手动指定。", + "实时进度：WebSocket 推送任务/步骤状态与速度。", + "详情：按步骤展示源 → 目标传输链路与错误信息。", ], }, ], diff --git a/frontend/web/src/views/module_ai/chat/index.vue b/frontend/web/src/views/module_ai/chat/index.vue index 8d51686d..9bf604f8 100644 --- a/frontend/web/src/views/module_ai/chat/index.vue +++ b/frontend/web/src/views/module_ai/chat/index.vue @@ -97,9 +97,10 @@ const connectWebSocket = () => { try { const url = new URL("/api/v1/ai/chat/ws", WS_URL); const token = Auth.getAccessToken(); - if (token) url.searchParams.append("token", token); - - ws = new WebSocket(url.toString()); + // 令牌经 Sec-WebSocket-Protocol 传递，避免出现在 URL 与服务端 access log 中 + ws = token + ? new WebSocket(url.toString(), ["access_token", `access_token.${token}`]) + : new WebSocket(url.toString()); ws.onopen = () => { isConnected.value = true; diff --git a/frontend/web/src/views/module_monitor/resource/index.vue b/frontend/web/src/views/module_monitor/resource/index.vue deleted file mode 100644 index c996d736..00000000 --- a/frontend/web/src/views/module_monitor/resource/index.vue +++ /dev/null @@ -1,652 +0,0 @@ - - - - - - diff --git a/frontend/web/src/views/module_storage/file/index.vue b/frontend/web/src/views/module_storage/file/index.vue deleted file mode 100644 index bd181025..00000000 --- a/frontend/web/src/views/module_storage/file/index.vue +++ /dev/null @@ -1,353 +0,0 @@ - - - - diff --git a/frontend/web/src/views/module_storage/source/index.vue b/frontend/web/src/views/module_storage/source/index.vue deleted file mode 100644 index f43c608a..00000000 --- a/frontend/web/src/views/module_storage/source/index.vue +++ /dev/null @@ -1,607 +0,0 @@ - - - - diff --git a/frontend/web/src/views/module_system/chat/index.vue b/frontend/web/src/views/module_system/chat/index.vue index 02c44f5d..91cfa8f5 100644 --- a/frontend/web/src/views/module_system/chat/index.vue +++ b/frontend/web/src/views/module_system/chat/index.vue @@ -1,7 +1,9 @@