From 25acc9427c43dfd93a3a4553e01f86d331613aef Mon Sep 17 00:00:00 2001 From: insistence <92962165+insistence@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:21:09 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E5=8A=9F=E8=83=BD=E5=B9=B6=E5=A2=9E=E5=BC=BA?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E6=9C=8D=E5=8A=A1=E5=AE=89=E5=85=A8=E6=80=A7?= =?UTF-8?q?=20(#111)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 新增文件管理功能，提升文件服务安全性 * docs: 新增文件管理使用文档 * docs: 更新README文档 * feat: 新增业务附件组件 * feat: 新增文件存储对账中心 * docs: 更新README文档 * feat: 新增到期文件处置功能 * perf: 完善上传人权限显式化 * feat: 新增Range 与断点下载 * docs: 更新README文档 --- README.md | 4 + .../src/utils/transportCryptoPolicy.js | 2 + ruoyi-fastapi-backend/.env.dev | 2 +- ruoyi-fastapi-backend/.env.dockermy | 2 +- ruoyi-fastapi-backend/.env.dockerpg | 2 +- ruoyi-fastapi-backend/.env.prod | 2 +- .../common/annotation/log_annotation.py | 9 +- .../common/aspect/data_scope.py | 4 +- ruoyi-fastapi-backend/common/constant.py | 16 + ruoyi-fastapi-backend/config/env.py | 13 +- .../docs/file_management_usage_guide.md | 383 ++++ ruoyi-fastapi-backend/exceptions/exception.py | 9 + ruoyi-fastapi-backend/exceptions/handle.py | 16 + .../middlewares/cors_middleware.py | 5 + .../controller/common_controller.py | 126 +- .../controller/file_controller.py | 915 +++++++++ .../module_admin/dao/file_access_dao.py | 366 ++++ .../module_admin/dao/file_business_dao.py | 645 +++++++ .../module_admin/dao/file_info_dao.py | 1209 ++++++++++++ .../module_admin/entity/do/file_do.py | 261 +++ .../module_admin/entity/vo/common_vo.py | 5 + .../module_admin/entity/vo/file_vo.py | 551 ++++++ .../module_admin/service/common_service.py | 544 +++++- .../service/file_access_service.py | 572 ++++++ .../service/file_business_service.py | 536 ++++++ .../module_admin/service/file_service.py | 1654 +++++++++++++++++ .../module_admin/service/log_service.py | 19 +- ruoyi-fastapi-backend/module_task/__init__.py | 2 +- .../module_task/file_task.py | 66 + .../scripts/migrate_legacy_files.py | 289 +++ .../sql/ruoyi-fastapi-pg.sql | 336 ++++ ruoyi-fastapi-backend/sql/ruoyi-fastapi.sql | 221 +++ .../sub_applications/staticfiles.py | 39 +- .../tests/test_common_file_security.py | 721 +++++++ .../tests/test_file_access_log.py | 165 ++ ruoyi-fastapi-backend/tests/test_file_acl.py | 580 ++++++ ...test_file_lifecycle_retention_execution.py | 402 ++++ .../tests/test_file_management.py | 876 +++++++++ .../tests/test_file_reconcile.py | 340 ++++ .../tests/test_file_reference.py | 291 +++ .../tests/test_file_retention_policy.py | 130 ++ .../tests/test_migrate_legacy_files.py | 169 ++ ruoyi-fastapi-backend/utils/file_util.py | 1019 ++++++++++ ruoyi-fastapi-backend/utils/response_util.py | 8 +- ruoyi-fastapi-backend/utils/upload_util.py | 221 ++- ruoyi-fastapi-frontend/src/api/system/file.js | 241 +++ .../components/BusinessFileUpload/index.vue | 565 ++++++ .../src/components/FileUpload/index.vue | 23 +- ruoyi-fastapi-frontend/src/main.js | 3 + .../src/plugins/download.js | 139 +- .../src/utils/transportCryptoPolicy.js | 4 +- .../system/file/components/FileAclDrawer.vue | 383 ++++ .../file/components/FileAuditDrawer.vue | 303 +++ .../file/components/FileDetailDialog.vue | 170 ++ .../file/components/FileReconcileDrawer.vue | 770 ++++++++ .../file/components/FileReferenceDrawer.vue | 106 ++ .../components/FileRetentionPolicyDrawer.vue | 248 +++ .../FileRetentionReminderDrawer.vue | 437 +++++ .../system/file/components/FileSearchForm.vue | 132 ++ .../system/file/components/FileStatistics.vue | 272 +++ .../system/file/components/FileTable.vue | 297 +++ .../file/components/FileTransferDialog.vue | 175 ++ .../system/file/components/fileFormatters.js | 146 ++ .../src/views/system/file/index.vue | 375 ++++ 64 files changed, 18444 insertions(+), 92 deletions(-) create mode 100644 ruoyi-fastapi-backend/docs/file_management_usage_guide.md create mode 100644 ruoyi-fastapi-backend/module_admin/controller/file_controller.py create mode 100644 ruoyi-fastapi-backend/module_admin/dao/file_access_dao.py create mode 100644 ruoyi-fastapi-backend/module_admin/dao/file_business_dao.py create mode 100644 ruoyi-fastapi-backend/module_admin/dao/file_info_dao.py create mode 100644 ruoyi-fastapi-backend/module_admin/entity/do/file_do.py create mode 100644 ruoyi-fastapi-backend/module_admin/entity/vo/file_vo.py create mode 100644 ruoyi-fastapi-backend/module_admin/service/file_access_service.py create mode 100644 ruoyi-fastapi-backend/module_admin/service/file_business_service.py create mode 100644 ruoyi-fastapi-backend/module_admin/service/file_service.py create mode 100644 ruoyi-fastapi-backend/module_task/file_task.py create mode 100644 ruoyi-fastapi-backend/scripts/migrate_legacy_files.py create mode 100644 ruoyi-fastapi-backend/tests/test_common_file_security.py create mode 100644 ruoyi-fastapi-backend/tests/test_file_access_log.py create mode 100644 ruoyi-fastapi-backend/tests/test_file_acl.py create mode 100644 ruoyi-fastapi-backend/tests/test_file_lifecycle_retention_execution.py create mode 100644 ruoyi-fastapi-backend/tests/test_file_management.py create mode 100644 ruoyi-fastapi-backend/tests/test_file_reconcile.py create mode 100644 ruoyi-fastapi-backend/tests/test_file_reference.py create mode 100644 ruoyi-fastapi-backend/tests/test_file_retention_policy.py create mode 100644 ruoyi-fastapi-backend/tests/test_migrate_legacy_files.py create mode 100644 ruoyi-fastapi-backend/utils/file_util.py create mode 100644 ruoyi-fastapi-frontend/src/api/system/file.js create mode 100644 ruoyi-fastapi-frontend/src/components/BusinessFileUpload/index.vue create mode 100644 ruoyi-fastapi-frontend/src/views/system/file/components/FileAclDrawer.vue create mode 100644 ruoyi-fastapi-frontend/src/views/system/file/components/FileAuditDrawer.vue create mode 100644 ruoyi-fastapi-frontend/src/views/system/file/components/FileDetailDialog.vue create mode 100644 ruoyi-fastapi-frontend/src/views/system/file/components/FileReconcileDrawer.vue create mode 100644 ruoyi-fastapi-frontend/src/views/system/file/components/FileReferenceDrawer.vue create mode 100644 ruoyi-fastapi-frontend/src/views/system/file/components/FileRetentionPolicyDrawer.vue create mode 100644 ruoyi-fastapi-frontend/src/views/system/file/components/FileRetentionReminderDrawer.vue create mode 100644 ruoyi-fastapi-frontend/src/views/system/file/components/FileSearchForm.vue create mode 100644 ruoyi-fastapi-frontend/src/views/system/file/components/FileStatistics.vue create mode 100644 ruoyi-fastapi-frontend/src/views/system/file/components/FileTable.vue create mode 100644 ruoyi-fastapi-frontend/src/views/system/file/components/FileTransferDialog.vue create mode 100644 ruoyi-fastapi-frontend/src/views/system/file/components/fileFormatters.js create mode 100644 ruoyi-fastapi-frontend/src/views/system/file/index.vue diff --git a/README.md b/README.md index 007b28e..9c881cd 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ RuoYi-Vue3-FastAPI是一套全部开源的快速开发平台，毫无保留给 17. 系统接口：根据业务代码自动生成相关的api接口文档。 18. 代码生成：配置数据库表信息一键生成前后端代码（python、sql、vue、js），支持下载。 19. AI管理：提供AI模型管理和AI对话功能。 +20. 文件管理：统一管理公开文件和受保护附件，支持访问控制、业务引用保护、操作审计、回收站、保留策略及存储对账。 ## 演示图 @@ -162,6 +163,9 @@ RuoYi-Vue3-FastAPI是一套全部开源的快速开发平台，毫无保留给 + + file + profile diff --git a/ruoyi-fastapi-app/src/utils/transportCryptoPolicy.js b/ruoyi-fastapi-app/src/utils/transportCryptoPolicy.js index 022d4ef..3f5a4fc 100644 --- a/ruoyi-fastapi-app/src/utils/transportCryptoPolicy.js +++ b/ruoyi-fastapi-app/src/utils/transportCryptoPolicy.js @@ -6,6 +6,8 @@ const EXCLUDED_URL_PATTERNS = [ "/transport/crypto/public-key", "/common/download", "/common/download/resource", + "/common/files/", + "/system/file/download/", ]; const TRANSPORT_FRONTEND_CONFIG_CACHE_KEY = "transportCryptoFrontendConfig"; const TRANSPORT_FRONTEND_CONFIG_URL = "/transport/crypto/frontend-config"; diff --git a/ruoyi-fastapi-backend/.env.dev b/ruoyi-fastapi-backend/.env.dev index b8e470d..4c56f23 100644 --- a/ruoyi-fastapi-backend/.env.dev +++ b/ruoyi-fastapi-backend/.env.dev @@ -170,4 +170,4 @@ TRANSPORT_CRYPTO_ENABLED_PATHS = '' # 强制要求传输层加密的路径列表，多个值使用逗号分隔 TRANSPORT_CRYPTO_REQUIRED_PATHS = '' # 排除传输层加密的路径列表，多个值使用逗号分隔 -TRANSPORT_CRYPTO_EXCLUDE_PATHS = '/openapi.json,/docs,/docs/oauth2-redirect,/redoc,/transport/crypto/frontend-config,/transport/crypto/public-key,/common/download,/common/download/resource' +TRANSPORT_CRYPTO_EXCLUDE_PATHS = '/openapi.json,/docs,/docs/oauth2-redirect,/redoc,/transport/crypto/frontend-config,/transport/crypto/public-key,/common/download,/common/download/resource,/common/files,/system/file/download' diff --git a/ruoyi-fastapi-backend/.env.dockermy b/ruoyi-fastapi-backend/.env.dockermy index db6df78..78e24a1 100644 --- a/ruoyi-fastapi-backend/.env.dockermy +++ b/ruoyi-fastapi-backend/.env.dockermy @@ -170,4 +170,4 @@ TRANSPORT_CRYPTO_ENABLED_PATHS = '' # 强制要求传输层加密的路径列表，多个值使用逗号分隔 TRANSPORT_CRYPTO_REQUIRED_PATHS = '' # 排除传输层加密的路径列表，多个值使用逗号分隔 -TRANSPORT_CRYPTO_EXCLUDE_PATHS = '/openapi.json,/docs,/docs/oauth2-redirect,/redoc,/transport/crypto/frontend-config,/transport/crypto/public-key,/common/download,/common/download/resource' +TRANSPORT_CRYPTO_EXCLUDE_PATHS = '/openapi.json,/docs,/docs/oauth2-redirect,/redoc,/transport/crypto/frontend-config,/transport/crypto/public-key,/common/download,/common/download/resource,/common/files,/system/file/download' diff --git a/ruoyi-fastapi-backend/.env.dockerpg b/ruoyi-fastapi-backend/.env.dockerpg index cf1e480..2f501e6 100644 --- a/ruoyi-fastapi-backend/.env.dockerpg +++ b/ruoyi-fastapi-backend/.env.dockerpg @@ -170,4 +170,4 @@ TRANSPORT_CRYPTO_ENABLED_PATHS = '' # 强制要求传输层加密的路径列表，多个值使用逗号分隔 TRANSPORT_CRYPTO_REQUIRED_PATHS = '' # 排除传输层加密的路径列表，多个值使用逗号分隔 -TRANSPORT_CRYPTO_EXCLUDE_PATHS = '/openapi.json,/docs,/docs/oauth2-redirect,/redoc,/transport/crypto/frontend-config,/transport/crypto/public-key,/common/download,/common/download/resource' +TRANSPORT_CRYPTO_EXCLUDE_PATHS = '/openapi.json,/docs,/docs/oauth2-redirect,/redoc,/transport/crypto/frontend-config,/transport/crypto/public-key,/common/download,/common/download/resource,/common/files,/system/file/download' diff --git a/ruoyi-fastapi-backend/.env.prod b/ruoyi-fastapi-backend/.env.prod index 4e4046c..241f32c 100644 --- a/ruoyi-fastapi-backend/.env.prod +++ b/ruoyi-fastapi-backend/.env.prod @@ -170,4 +170,4 @@ TRANSPORT_CRYPTO_ENABLED_PATHS = '' # 强制要求传输层加密的路径列表，多个值使用逗号分隔 TRANSPORT_CRYPTO_REQUIRED_PATHS = '' # 排除传输层加密的路径列表，多个值使用逗号分隔 -TRANSPORT_CRYPTO_EXCLUDE_PATHS = '/openapi.json,/docs,/docs/oauth2-redirect,/redoc,/transport/crypto/frontend-config,/transport/crypto/public-key,/common/download,/common/download/resource' +TRANSPORT_CRYPTO_EXCLUDE_PATHS = '/openapi.json,/docs,/docs/oauth2-redirect,/redoc,/transport/crypto/frontend-config,/transport/crypto/public-key,/common/download,/common/download/resource,/common/files,/system/file/download' diff --git a/ruoyi-fastapi-backend/common/annotation/log_annotation.py b/ruoyi-fastapi-backend/common/annotation/log_annotation.py index 4b11b59..da61900 100644 --- a/ruoyi-fastapi-backend/common/annotation/log_annotation.py +++ b/ruoyi-fastapi-backend/common/annotation/log_annotation.py @@ -19,7 +19,12 @@ from user_agents import parse from common.context import RequestContext from common.enums import BusinessType from config.env import AppConfig -from exceptions.exception import LoginException, ServiceException, ServiceWarning +from exceptions.exception import ( + FileRangeNotSatisfiableException, + LoginException, + ServiceException, + ServiceWarning, +) from module_admin.entity.vo.log_vo import LogininforModel, OperLogModel from module_admin.service.log_service import LogQueueService from utils.client_ip_util import ClientIPUtil @@ -196,6 +201,8 @@ class Log: except ServiceException as e: logger.error(e.message) result = ResponseUtil.error(data=e.data, msg=e.message) + except FileRangeNotSatisfiableException: + raise except Exception as e: logger.exception(e) result = ResponseUtil.error(msg=str(e)) diff --git a/ruoyi-fastapi-backend/common/aspect/data_scope.py b/ruoyi-fastapi-backend/common/aspect/data_scope.py index 4d3f93a..80d8304 100644 --- a/ruoyi-fastapi-backend/common/aspect/data_scope.py +++ b/ruoyi-fastapi-backend/common/aspect/data_scope.py @@ -69,7 +69,7 @@ class GetDataScope: elif role.data_scope == self.DATA_SCOPE_DEPT: param_sql_list.append( getattr(self.query_alias, self.dept_alias) == dept_id - if hasattr(self.query_alias, self.dept_alias) + if dept_id is not None and hasattr(self.query_alias, self.dept_alias) else False ) elif role.data_scope == self.DATA_SCOPE_DEPT_AND_CHILD: @@ -79,7 +79,7 @@ class GetDataScope: or_(SysDept.dept_id == dept_id, func.find_in_set(dept_id, SysDept.ancestors)) ) ) - if hasattr(self.query_alias, self.dept_alias) + if dept_id is not None and hasattr(self.query_alias, self.dept_alias) else False ) elif role.data_scope == self.DATA_SCOPE_SELF: diff --git a/ruoyi-fastapi-backend/common/constant.py b/ruoyi-fastapi-backend/common/constant.py index 1d8cf23..4385cff 100644 --- a/ruoyi-fastapi-backend/common/constant.py +++ b/ruoyi-fastapi-backend/common/constant.py @@ -203,6 +203,12 @@ class ApiNamespace: SYSTEM_MENU_DETAIL: 菜单详情接口命名空间 SYSTEM_NOTICE_LIST: 通知公告列表接口命名空间 SYSTEM_NOTICE_DETAIL: 通知公告详情接口命名空间 + SYSTEM_FILE_DOWNLOAD: 文件管理下载接口命名空间 + SYSTEM_FILE_DELETE: 文件管理删除接口命名空间 + SYSTEM_FILE_RETENTION_POLICY: 文件业务保留策略接口命名空间 + SYSTEM_FILE_RECONCILE: 文件存储对账接口命名空间 + SYSTEM_FILE_TRANSFER: 文件管理转移接口命名空间 + SYSTEM_FILE_RESTORE: 文件管理恢复接口命名空间 SYSTEM_POST_LIST: 岗位列表接口命名空间 SYSTEM_POST_DETAIL: 岗位详情接口命名空间 SYSTEM_POST_EXPORT: 岗位导出接口命名空间 @@ -247,6 +253,8 @@ class ApiNamespace: LOGIN_USER_ROUTERS = 'login:user:routers' CAPTCHA_IMAGE = 'captcha:image' COMMON_UPLOAD = 'common:upload' + COMMON_PRIVATE_UPLOAD = 'common:private-upload' + COMMON_FILE_DOWNLOAD = 'common:file-download' TRANSPORT_CRYPTO_PUBLIC_KEY = 'transport-crypto:public-key' TRANSPORT_CRYPTO_FRONTEND_CONFIG = 'transport-crypto:frontend-config' @@ -297,6 +305,14 @@ class ApiNamespace: SYSTEM_NOTICE_LIST = 'system:notice:list' SYSTEM_NOTICE_DETAIL = 'system:notice:detail' + SYSTEM_FILE_DOWNLOAD = 'system:file:download' + SYSTEM_FILE_DELETE = 'system:file:delete' + SYSTEM_FILE_ACL = 'system:file:acl' + SYSTEM_FILE_RETENTION_POLICY = 'system:file:retention-policy' + SYSTEM_FILE_RECONCILE = 'system:file:reconcile' + SYSTEM_FILE_TRANSFER = 'system:file:transfer' + SYSTEM_FILE_RESTORE = 'system:file:restore' + SYSTEM_POST_LIST = 'system:post:list' SYSTEM_POST_DETAIL = 'system:post:detail' SYSTEM_POST_EXPORT = 'system:post:export' diff --git a/ruoyi-fastapi-backend/config/env.py b/ruoyi-fastapi-backend/config/env.py index 3dd6604..25a7e68 100644 --- a/ruoyi-fastapi-backend/config/env.py +++ b/ruoyi-fastapi-backend/config/env.py @@ -140,7 +140,8 @@ class TransportCryptoSettings(BaseSettings): transport_crypto_required_paths: str = '' transport_crypto_exclude_paths: str = ( '/openapi.json,/docs,/docs/oauth2-redirect,/redoc,' - '/transport/crypto/frontend-config,/transport/crypto/public-key,/common/download,/common/download/resource' + '/transport/crypto/frontend-config,/transport/crypto/public-key,/common/download,/common/download/resource,' + '/common/files,/system/file/download' ) @@ -169,6 +170,9 @@ class UploadSettings: UPLOAD_PREFIX = '/profile' UPLOAD_PATH = 'vf_admin/upload_path' + PRIVATE_UPLOAD_PATH = 'vf_admin/private_upload_path' + FILE_TRASH_PATH = 'vf_admin/file_trash_path' + FILE_RECONCILE_QUARANTINE_PATH = 'vf_admin/file_reconcile_quarantine_path' UPLOAD_MACHINE = 'A' DEFAULT_ALLOWED_EXTENSION = [ # 图片 @@ -200,10 +204,17 @@ class UploadSettings: 'pdf', ] DOWNLOAD_PATH = 'vf_admin/download_path' + MAX_FILE_SIZE = 100 * 1024 * 1024 def __init__(self) -> None: if not os.path.exists(self.UPLOAD_PATH): os.makedirs(self.UPLOAD_PATH) + if not os.path.exists(self.PRIVATE_UPLOAD_PATH): + os.makedirs(self.PRIVATE_UPLOAD_PATH) + if not os.path.exists(self.FILE_TRASH_PATH): + os.makedirs(self.FILE_TRASH_PATH) + if not os.path.exists(self.FILE_RECONCILE_QUARANTINE_PATH): + os.makedirs(self.FILE_RECONCILE_QUARANTINE_PATH) if not os.path.exists(self.DOWNLOAD_PATH): os.makedirs(self.DOWNLOAD_PATH) diff --git a/ruoyi-fastapi-backend/docs/file_management_usage_guide.md b/ruoyi-fastapi-backend/docs/file_management_usage_guide.md new file mode 100644 index 0000000..20297ee --- /dev/null +++ b/ruoyi-fastapi-backend/docs/file_management_usage_guide.md @@ -0,0 +1,383 @@ +# 文件管理使用与业务接入指南 + +本文说明文件管理功能怎么使用，以及业务模块怎么接入。 + +## 1. 怎么选择上传方式 + +| 场景 | 使用方式 | 业务保存内容 | +| --- | --- | --- | +| 头像、Logo、富文本图片等公开资源 | `/common/upload` 或 `ImageUpload` | `/profile/...` URL | +| 旧业务的普通公开附件 | 默认 `FileUpload` | URL 字符串 | +| 简单私有附件，不需要业务引用保护 | `FileUpload :is-private="true"` | 鉴权下载 URL | +| 合同、审批材料等正式业务附件 | `BusinessFileUpload` | `{ fileId, name, url }` 列表 | + +正式业务附件推荐使用最后一种方式，它支持： + +- 文件下载鉴权。 +- 用户、角色、部门 ACL。 +- 业务引用保护，防止文件仍在使用时被删除。 +- 按业务类型应用保留策略。 +- 文件操作审计和回收站。 + +公开文件通过 `/profile` 直接访问，不适合存放需要权限保护的内容。 + +## 2. 文件管理页面怎么使用 + +文件管理页面主要用于： + +- 查询文件、占用空间、所有者、所属部门和文件状态。 +- 查看文件详情、业务引用和操作审计。 +- 为私有文件授权用户、角色或部门下载。 +- 转移文件所有者和所属部门。 +- 将没有业务引用的文件移入回收站。 +- 恢复文件或永久清理回收站文件。 +- 配置业务保留策略和查看到期提醒。 +- 执行存储对账并处理文件缺失、错位、孤立或内容不一致。 + +文件管理操作同时受菜单权限和数据权限限制。部门管理员只能管理其文件数据范围内的文件。 + +私有文件下载规则可以简单理解为： + +1. 文件到期后直接拒绝。 +2. 管理员或文件所有者允许。 +3. 命中的 `deny` ACL 优先拒绝。 +4. 上传者允许。 +5. 命中的 `allow` ACL 允许。 +6. 其他情况默认拒绝。 + +### 2.1 存储对账怎么使用 + +系统管理员在文件管理页面点击“对账”，可以执行文件信息表与本地存储的双向检查。默认校验文件是否存在、所在区域和大小；需要确认文件内容没有被替换时，再开启 SHA-256 校验。摘要校验会读取全部文件内容，建议在低峰期执行。 + +异常明细会给出预期位置、实际位置和服务端允许的处理动作。常用处理方式如下： + +- 文件位于错误区域时移动到预期区域。 +- 有效文件误入回收站时恢复，已删除文件仍在正式区时移入回收站。 +- 重复文件或孤立文件先隔离，核实后恢复或永久删除。 +- 确认当前物理文件正确时，接受当前大小和摘要。 +- 确认孤立文件属于系统时，将其登记为当前管理员所有的文件。 +- 暂时无需处理时忽略，之后可以重新打开。 + +所有处理都必须填写原因。隔离区不提供静态访问；永久删除隔离文件不可恢复。定时任务中的“文件存储对账”默认暂停，可在任务管理中确认运行时段后启用。 + +## 3. 前端接入 + +### 3.1 通用约定 + +Vue2 和 Vue3 使用相同的受保护文件上传接口： + +```http +POST /common/files/upload +Content-Type: multipart/form-data +Authorization: Bearer +``` + +上传成功后会返回： + +```json +{ + "code": 200, + "fileId": "8e5787b4-daf7-4e31-bf04-f1cc16e0f65a", + "originalFilename": "合同.pdf", + "accessType": "private", + "downloadUrl": "/common/files/8e5787b4-daf7-4e31-bf04-f1cc16e0f65a/download/合同.pdf" +} +``` + +业务表单都应保存 `{ fileId, name, url }`，提交业务接口时传完整的 `fileId` 列表。不要从下载 URL 中截取 `fileId`。 + +### 3.2 Vue3 写法 + +Vue3 项目已经全局注册 `BusinessFileUpload`： + +```vue + + + +``` + +### 3.3 Vue2 写法 + +Vue2 项目同样全局注册 `BusinessFileUpload`： + +```vue + + + +``` + +组件支持结构化数据回显、上传失败重试、删除、拖动排序和鉴权下载。`v-model` 的数据格式始终为： + +```json +[ + { + "fileId": "8e5787b4-daf7-4e31-bf04-f1cc16e0f65a", + "name": "合同.pdf", + "url": "/common/files/8e5787b4-daf7-4e31-bf04-f1cc16e0f65a/download/存储文件名.pdf" + } +] +``` + +也可以监听 `change` 事件，事件参数依次为结构化文件列表和文件 ID 列表。 + +### 3.4 上传组件的适用范围 + +Vue2 和 Vue3 都可以使用下面的写法上传简单私有附件： + +```vue + +``` + +两个版本的 `FileUpload` 都只把下载 URL 写入 `v-model`，不会保留 `fileId`。因此它们只适合旧业务或不需要引用保护的简单附件。正式业务附件统一使用 `BusinessFileUpload`。 + +两个版本的 `ImageUpload` 都继续用于公开图片。受保护文件下载应调用 `$download.file()`，不要使用普通 `` 标签直接打开。 + +### 3.5 Range 与断点下载 + +Vue2 和 Vue3 的 `$download.file()`、`$download.resource()` 会按 8 MB 分段下载，并在分段请求失败时从当前分段重新请求。业务组件继续使用原有调用方式，不需要自行处理 `Range`。 + +自定义客户端可以向业务附件下载、文件管理下载或资源下载接口发送标准单区间请求： + +```http +Range: bytes=8388608-16777215 +``` + +服务端返回 `206 Partial Content`，并携带 `Accept-Ranges`、`Content-Range` 和 `Content-Length`。起止范围、开放结束范围和后缀范围均受支持；不支持多区间请求，无效或越界范围返回 `416`。 + +`/common/download?delete=true` 用于下载后删除临时导出文件，始终整文件返回，不参与断点下载。需要断点下载时必须使用不会在响应结束后删除文件的下载地址。 + +## 4. 后端业务接入 + +下面以“合同”业务为例。 + +### 4.1 定义业务类型 + +每个业务模块定义一个稳定的业务类型： + +```python +CONTRACT_FILE_BUSINESS_TYPE = 'contract' +``` + +该值用于关联业务引用和保留策略。上线后不要随意修改，也不要使用中文显示名称。 + +### 4.2 请求模型接收文件 ID + +```python +class ContractModel(BaseModel): + model_config = ConfigDict(alias_generator=to_camel) + + contract_id: int | None = Field(default=None, description='合同ID') + contract_name: str = Field(description='合同名称') + attachment_file_ids: list[str] = Field( + default_factory=list, + max_length=100, + description='附件文件ID列表', + ) +``` + +修改接口必须传“修改后需要保留的完整文件 ID 列表”，空列表表示移除全部附件。 + +### 4.3 注入文件数据权限 + +业务控制器需要注入文件数据权限，并传给业务 Service： + +```python +file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency( + SysFileInfo, + user_alias='owner_user_id', + dept_alias='dept_id', + ), +] +``` + +引用服务会校验文件存在、状态正常，并且位于当前用户的文件数据范围内。 + +### 4.4 新增和修改时同步引用 + +新增业务记录取得业务 ID 后，在同一个事务中写入文件引用： + +```python +try: + db_contract = await ContractDao.add_contract_dao(query_db, contract) + await FileReferenceService.replace_business_file_references_services( + query_db=query_db, + business_type=CONTRACT_FILE_BUSINESS_TYPE, + business_id=str(db_contract.contract_id), + file_ids=contract.attachment_file_ids, + create_by=current_user.user.user_name, + file_data_scope_sql=file_data_scope_sql, + business_name=contract.contract_name, + ) + await query_db.commit() +except Exception: + await query_db.rollback() + raise +``` + +修改业务时调用同一个方法，传入修改后的完整列表： + +```python +await ContractDao.edit_contract_dao(query_db, contract) +await FileReferenceService.replace_business_file_references_services( + query_db=query_db, + business_type=CONTRACT_FILE_BUSINESS_TYPE, + business_id=str(contract.contract_id), + file_ids=contract.attachment_file_ids, + create_by=current_user.user.user_name, + file_data_scope_sql=file_data_scope_sql, + business_name=contract.contract_name, +) +await query_db.commit() +``` + +`replace_business_file_references_services` 是全量替换： + +- 原来是 `[A, B]`，现在传 `[B, C]`：解除 A，保留 B，新增 C。 +- 传空列表：解除该业务对象的全部引用。 + +业务 DAO 应使用 `flush()` 获取新增 ID，不要提前 `commit()`。业务数据和文件引用必须由业务 Service 统一提交或回滚。 + +### 4.5 删除业务时解除引用 + +先用业务模块自身的数据权限确认业务对象可以删除，再在同一事务中解除引用： + +```python +try: + db_contract = await ContractDao.get_contract_by_id_for_update( + query_db, + contract_id, + contract_data_scope_sql, + ) + if db_contract is None: + raise ServiceException(message='合同不存在或超出数据权限') + + await FileReferenceService.remove_business_file_references_services( + query_db, + CONTRACT_FILE_BUSINESS_TYPE, + str(contract_id), + ) + await ContractDao.delete_contract_dao(query_db, contract_id) + await query_db.commit() +except Exception: + await query_db.rollback() + raise +``` + +`remove_business_file_references_services` 不负责校验合同权限，所以必须在业务模块完成鉴权后调用。 + +解除引用不会立即删除文件。没有其他业务引用后，文件管理员才能将文件移入回收站。 + +### 4.6 业务详情回显附件 + +业务详情接口应按以下条件查询 `sys_file_reference` 并关联 `sys_file_info`： + +```text +business_type = CONTRACT_FILE_BUSINESS_TYPE +business_id = str(contract_id) +``` + +建议返回： + +```json +{ + "fileId": "8e5787b4-daf7-4e31-bf04-f1cc16e0f65a", + "name": "合同.pdf", + "downloadUrl": "/common/files/8e5787b4-daf7-4e31-bf04-f1cc16e0f65a/download/合同.pdf" +} +``` + +不要向前端返回 `storage_key`、物理路径或私有目录信息。 + +## 5. 业务引用、下载权限和保留策略 + +这三项作用不同： + +| 能力 | 作用 | +| --- | --- | +| 业务引用 | 记录文件正在被哪个业务使用，并阻止误删 | +| 文件 ACL | 决定除所有者、上传者外，还有谁可以下载 | +| 保留策略 | 决定新业务引用什么时候到期 | + +业务引用不会自动授予下载权限。其他能查看合同的用户如果也要下载附件，需要： + +- 在文件管理页面手工配置用户、角色或部门 ACL；或 +- 由业务模块在参与人、角色、部门变化时同步文件 ACL。 + +保留策略按 `business_type` 生效，不是在文件上直接选择策略。例如： + +```text +保留策略：business_type=contract，retention_days=365 +业务引用：business_type=contract，business_id=1001 +结果：该引用在创建时得到 365 天的保留期限 +``` + +因此，只有合同模块实际调用引用服务，并传入相同的 `business_type='contract'`，策略才会作用到附件。 + +还需要注意： + +- 策略只应用于新建或重新写入的引用，不会自动修改历史引用。 +- 配置保留策略的业务只能引用私有文件。 +- 到期后文件不能下载，但不会自动删除，也不会自动解除引用。 +- 一个文件有多个引用时，只要存在永久引用，文件就不会到期。 + +## 6. 接入检查 + +一个业务模块完成以下内容即视为接入完成： + +- [ ] 使用 `/common/files/upload` 上传正式业务附件。 +- [ ] 前端保存上传响应中的 `fileId`。 +- [ ] 新增和修改接口传递完整的文件 ID 列表。 +- [ ] 控制器注入文件数据权限。 +- [ ] 业务新增、修改与引用更新使用同一个事务。 +- [ ] 业务删除先鉴权，再在同一事务中解除引用。 +- [ ] 业务详情返回结构化附件和鉴权下载地址。 +- [ ] 明确非所有者用户通过什么 ACL 下载。 +- [ ] 如果配置保留策略，策略的 `business_type` 与代码常量一致。 + +完成这些步骤后，文件管理页面才能正确显示业务引用，删除保护和保留策略也才会真正生效。 diff --git a/ruoyi-fastapi-backend/exceptions/exception.py b/ruoyi-fastapi-backend/exceptions/exception.py index 02580ed..519df0e 100644 --- a/ruoyi-fastapi-backend/exceptions/exception.py +++ b/ruoyi-fastapi-backend/exceptions/exception.py @@ -48,6 +48,15 @@ class ServiceWarning(Exception): self.message = message +class FileRangeNotSatisfiableException(Exception): + """ + 文件Range范围不可满足异常 + """ + + def __init__(self, file_size: int) -> None: + self.file_size = file_size + + class ModelValidatorException(Exception): """ 自定义模型校验异常ModelValidatorException diff --git a/ruoyi-fastapi-backend/exceptions/handle.py b/ruoyi-fastapi-backend/exceptions/handle.py index 3e8e590..6c463ef 100644 --- a/ruoyi-fastapi-backend/exceptions/handle.py +++ b/ruoyi-fastapi-backend/exceptions/handle.py @@ -4,6 +4,7 @@ from pydantic_validation_decorator import FieldValidationError from exceptions.exception import ( AuthException, + FileRangeNotSatisfiableException, LoginException, ModelValidatorException, PermissionException, @@ -58,6 +59,21 @@ def handle_exception(app: FastAPI) -> None: logger.warning(exc.message) return ResponseUtil.failure(data=exc.data, msg=exc.message) + # 文件Range范围不可满足异常 + @app.exception_handler(FileRangeNotSatisfiableException) + async def file_range_not_satisfiable_exception_handler( + request: Request, + exc: FileRangeNotSatisfiableException, + ) -> Response: + return Response( + status_code=416, + headers={ + 'Accept-Ranges': 'bytes', + 'Content-Range': f'bytes */{exc.file_size}', + 'Content-Length': '0', + }, + ) + # 处理其他http请求异常 @app.exception_handler(HTTPException) async def http_exception_handler(request: Request, exc: HTTPException) -> Response: diff --git a/ruoyi-fastapi-backend/middlewares/cors_middleware.py b/ruoyi-fastapi-backend/middlewares/cors_middleware.py index df6a0ce..42206a6 100644 --- a/ruoyi-fastapi-backend/middlewares/cors_middleware.py +++ b/ruoyi-fastapi-backend/middlewares/cors_middleware.py @@ -15,6 +15,11 @@ def add_cors_middleware(app: FastAPI) -> None: 'x-body-encrypted', 'x-key-id', 'x-encrypt-alg', + 'download-filename', + 'content-disposition', + 'accept-ranges', + 'content-range', + 'content-length', ] # 后台api允许跨域 diff --git a/ruoyi-fastapi-backend/module_admin/controller/common_controller.py b/ruoyi-fastapi-backend/module_admin/controller/common_controller.py index bfa4282..3b165c8 100644 --- a/ruoyi-fastapi-backend/module_admin/controller/common_controller.py +++ b/ruoyi-fastapi-backend/module_admin/controller/common_controller.py @@ -1,17 +1,22 @@ from typing import Annotated +from uuid import UUID from fastapi import BackgroundTasks, File, Query, Request, Response, UploadFile from fastapi.responses import StreamingResponse +from sqlalchemy.ext.asyncio import AsyncSession from common.annotation.rate_limit_annotation import ApiRateLimit, ApiRateLimitPreset -from common.aspect.pre_auth import PreAuthDependency +from common.aspect.db_seesion import DBSessionDependency +from common.aspect.pre_auth import CurrentUserDependency, PreAuthDependency from common.constant import ApiNamespace from common.router import APIRouterPro from common.vo import DynamicResponseModel from module_admin.entity.vo.common_vo import UploadResponseModel +from module_admin.entity.vo.user_vo import CurrentUserModel from module_admin.service.common_service import CommonService from utils.log_util import logger from utils.response_util import ResponseUtil +from utils.upload_util import UploadUtil common_controller = APIRouterPro(prefix='/common', order_num=16, tags=['通用模块'], dependencies=[PreAuthDependency()]) @@ -23,13 +28,82 @@ common_controller = APIRouterPro(prefix='/common', order_num=16, tags=['通用 response_model=DynamicResponseModel[UploadResponseModel], ) @ApiRateLimit(namespace=ApiNamespace.COMMON_UPLOAD, preset=ApiRateLimitPreset.COMMON_UPLOAD) -async def common_upload(request: Request, file: Annotated[UploadFile, File(...)]) -> Response: - upload_result = await CommonService.upload_service(request, file) +async def common_upload( + request: Request, + file: Annotated[UploadFile, File(...)], + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], +) -> Response: + upload_result = await CommonService.upload_service(request, query_db, current_user, file, access_type='public') logger.info('上传成功') return ResponseUtil.success(model_content=upload_result.result) +@common_controller.post( + '/files/upload', + summary='受保护文件上传接口', + description='用于上传仅授权用户可以访问的受保护文件', + response_model=DynamicResponseModel[UploadResponseModel], +) +@ApiRateLimit(namespace=ApiNamespace.COMMON_PRIVATE_UPLOAD, preset=ApiRateLimitPreset.USER_RESOURCE_UPLOAD) +async def common_private_upload( + request: Request, + file: Annotated[UploadFile, File(...)], + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], +) -> Response: + upload_result = await CommonService.upload_service(request, query_db, current_user, file, access_type='private') + logger.info('受保护文件上传成功') + + return ResponseUtil.success(model_content=upload_result.result) + + +@common_controller.get( + '/files/{file_id}/download/{display_name}', + summary='已登记文件下载接口', + description='用于根据文件ID下载已登记的公开或受保护文件', + response_class=StreamingResponse, + responses={ + 200: { + 'description': '流式返回文件', + 'content': { + 'application/octet-stream': {}, + }, + }, + 206: {'description': '分段返回文件'}, + 416: {'description': '请求的字节范围不可满足'}, + }, +) +@ApiRateLimit(namespace=ApiNamespace.COMMON_FILE_DOWNLOAD, preset=ApiRateLimitPreset.USER_RESOURCE_DOWNLOAD) +async def common_managed_file_download( + request: Request, + file_id: UUID, + display_name: str, + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], +) -> Response: + download_result = await CommonService.download_managed_file_services( + request, + query_db, + current_user, + str(file_id), + range_header=request.headers.get('Range'), + ) + logger.info(f'文件{file_id}下载成功') + + return ResponseUtil.streaming( + data=download_result.data, + headers=UploadUtil.build_download_headers( + download_result.filename, + download_result.byte_range, + download_result.accept_ranges, + ), + media_type='application/octet-stream', + status_code=206 if download_result.byte_range.is_partial else 200, + ) + + @common_controller.get( '/download', summary='通用文件下载接口', @@ -41,7 +115,9 @@ async def common_upload(request: Request, file: Annotated[UploadFile, File(...)] 'content': { 'application/octet-stream': {}, }, - } + }, + 206: {'description': '分段返回文件'}, + 416: {'description': '请求的字节范围不可满足'}, }, ) async def common_download( @@ -50,10 +126,24 @@ async def common_download( file_name: Annotated[str, Query(alias='fileName')], delete: Annotated[bool, Query()], ) -> Response: - download_result = await CommonService.download_services(background_tasks, file_name, delete) - logger.info(download_result.message) + download_result = await CommonService.download_services( + background_tasks, + file_name, + delete, + range_header=request.headers.get('Range'), + ) + logger.info('下载成功') - return ResponseUtil.streaming(data=download_result.result) + return ResponseUtil.streaming( + data=download_result.data, + headers=UploadUtil.build_download_headers( + download_result.filename, + download_result.byte_range, + download_result.accept_ranges, + ), + media_type='application/octet-stream', + status_code=206 if download_result.byte_range.is_partial else 200, + ) @common_controller.get( @@ -67,11 +157,25 @@ async def common_download( 'content': { 'application/octet-stream': {}, }, - } + }, + 206: {'description': '分段返回文件'}, + 416: {'description': '请求的字节范围不可满足'}, }, ) async def common_download_resource(request: Request, resource: Annotated[str, Query()]) -> Response: - download_resource_result = await CommonService.download_resource_services(resource) - logger.info(download_resource_result.message) + download_result = await CommonService.download_resource_services( + resource, + range_header=request.headers.get('Range'), + ) + logger.info('下载成功') - return ResponseUtil.streaming(data=download_resource_result.result) + return ResponseUtil.streaming( + data=download_result.data, + headers=UploadUtil.build_download_headers( + download_result.filename, + download_result.byte_range, + download_result.accept_ranges, + ), + media_type='application/octet-stream', + status_code=206 if download_result.byte_range.is_partial else 200, + ) diff --git a/ruoyi-fastapi-backend/module_admin/controller/file_controller.py b/ruoyi-fastapi-backend/module_admin/controller/file_controller.py new file mode 100644 index 0000000..f0870e7 --- /dev/null +++ b/ruoyi-fastapi-backend/module_admin/controller/file_controller.py @@ -0,0 +1,915 @@ +from typing import Annotated, Literal +from uuid import UUID + +from fastapi import BackgroundTasks, Path, Query, Request, Response +from fastapi.responses import StreamingResponse +from sqlalchemy import ColumnElement +from sqlalchemy.ext.asyncio import AsyncSession + +from common.annotation.log_annotation import Log +from common.annotation.rate_limit_annotation import ApiRateLimit, ApiRateLimitPreset +from common.aspect.data_scope import DataScopeDependency +from common.aspect.db_seesion import DBSessionDependency +from common.aspect.interface_auth import UserInterfaceAuthDependency +from common.aspect.pre_auth import CurrentUserDependency, PreAuthDependency +from common.constant import ApiNamespace +from common.enums import BusinessType +from common.router import APIRouterPro +from common.vo import DataResponseModel, PageResponseModel, ResponseBaseModel +from module_admin.entity.do.dept_do import SysDept +from module_admin.entity.do.file_do import SysFileInfo +from module_admin.entity.do.user_do import SysUser +from module_admin.entity.vo.dept_vo import DeptTreeModel +from module_admin.entity.vo.file_vo import ( + BatchSaveFileAclModel, + DeleteFileModel, + DisposeExpiredFileModel, + ExtendFileRetentionModel, + FileAccessLogModel, + FileAccessLogPageQueryModel, + FileAclListModel, + FileAclSubjectOptionModel, + FileInfoDisplayModel, + FileInfoPageQueryModel, + FileReconcileHandleModel, + FileReconcileIssueModel, + FileReconcileIssuePageQueryModel, + FileReconcileRunModel, + FileReconcileRunPageQueryModel, + FileReconcileStartModel, + FileReconcileStatsModel, + FileReferenceModel, + FileRetentionNoticeModel, + FileRetentionNoticePageQueryModel, + FileRetentionPolicyModel, + FileRetentionScanModel, + FileStatsModel, + SaveFileAclModel, + TransferFileModel, +) +from module_admin.entity.vo.user_vo import CurrentUserModel +from module_admin.service.common_service import CommonService +from module_admin.service.file_access_service import FileAclService +from module_admin.service.file_business_service import ( + FileReferenceService, + FileRetentionNoticeService, + FileRetentionPolicyService, +) +from module_admin.service.file_service import ( + FileLifecycleService, + FileQueryService, + FileReconcileService, + FileRetentionDispositionService, + FileTransferService, +) +from utils.log_util import logger +from utils.response_util import ResponseUtil +from utils.upload_util import UploadUtil + +file_controller = APIRouterPro( + prefix='/system/file', order_num=11, tags=['系统管理-文件管理'], dependencies=[PreAuthDependency()] +) + + +@file_controller.get( + '/list', + summary='获取文件分页列表接口', + description='用于获取文件分页列表', + response_model=PageResponseModel[FileInfoDisplayModel], + dependencies=[UserInterfaceAuthDependency('system:file:list')], +) +async def get_system_file_list( + request: Request, + file_page_query: Annotated[FileInfoPageQueryModel, Query()], + query_db: Annotated[AsyncSession, DBSessionDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], +) -> Response: + file_page_query_result = await FileQueryService.get_file_list_services( + query_db, + file_page_query, + file_data_scope_sql, + is_page=True, + ) + logger.info('获取成功') + + return ResponseUtil.success(model_content=file_page_query_result) + + +@file_controller.get( + '/stats', + summary='获取文件管理统计接口', + description='用于获取当前数据范围和查询条件下的文件统计信息', + response_model=DataResponseModel[FileStatsModel], + dependencies=[UserInterfaceAuthDependency('system:file:list')], +) +async def get_system_file_stats( + request: Request, + file_page_query: Annotated[FileInfoPageQueryModel, Query()], + query_db: Annotated[AsyncSession, DBSessionDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], +) -> Response: + file_stats_result = await FileQueryService.get_file_stats_services( + query_db, + file_page_query, + file_data_scope_sql, + ) + logger.info('文件统计获取成功') + + return ResponseUtil.success(data=file_stats_result) + + +@file_controller.get( + '/reconcile/issues/list', + summary='获取文件存储对账异常分页列表接口', + description='用于获取文件存储对账异常和可用处理动作', + response_model=PageResponseModel[FileReconcileIssueModel], + dependencies=[UserInterfaceAuthDependency('system:file:reconcile')], +) +async def get_system_file_reconcile_issue_list( + request: Request, + issue_page_query: Annotated[FileReconcileIssuePageQueryModel, Query()], + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], +) -> Response: + issue_page_query_result = await FileReconcileService.get_reconcile_issue_list_services( + query_db, + current_user, + issue_page_query, + is_page=True, + ) + logger.info('文件存储对账异常列表获取成功') + + return ResponseUtil.success(model_content=issue_page_query_result) + + +@file_controller.get( + '/reconcile/runs/list', + summary='获取文件存储对账任务分页列表接口', + description='用于获取文件存储对账任务执行记录', + response_model=PageResponseModel[FileReconcileRunModel], + dependencies=[UserInterfaceAuthDependency('system:file:reconcile')], +) +async def get_system_file_reconcile_run_list( + request: Request, + run_page_query: Annotated[FileReconcileRunPageQueryModel, Query()], + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], +) -> Response: + run_page_query_result = await FileReconcileService.get_reconcile_run_list_services( + query_db, + current_user, + run_page_query, + is_page=True, + ) + logger.info('文件存储对账任务列表获取成功') + + return ResponseUtil.success(model_content=run_page_query_result) + + +@file_controller.get( + '/reconcile/stats', + summary='获取文件存储对账统计接口', + description='用于获取待处理异常和最近任务统计', + response_model=DataResponseModel[FileReconcileStatsModel], + dependencies=[UserInterfaceAuthDependency('system:file:reconcile')], +) +async def get_system_file_reconcile_stats( + request: Request, + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], +) -> Response: + reconcile_stats = await FileReconcileService.get_reconcile_stats_services( + query_db, + current_user, + ) + logger.info('文件存储对账统计获取成功') + + return ResponseUtil.success(data=reconcile_stats) + + +@file_controller.post( + '/reconcile/run', + summary='启动文件存储对账任务接口', + description='用于启动数据库和本地文件系统双向对账任务', + response_model=DataResponseModel[FileReconcileRunModel], + dependencies=[UserInterfaceAuthDependency('system:file:reconcile')], +) +@ApiRateLimit(namespace=ApiNamespace.SYSTEM_FILE_RECONCILE, preset=ApiRateLimitPreset.USER_DESTRUCTIVE_MUTATION) +@Log(title='文件存储对账', business_type=BusinessType.UPDATE) +async def start_system_file_reconcile( + request: Request, + start_reconcile: FileReconcileStartModel, + background_tasks: BackgroundTasks, + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], +) -> Response: + reconcile_run = await FileReconcileService.start_reconcile_run_services( + query_db, + check_hash=start_reconcile.check_hash, + current_user=current_user, + ) + background_tasks.add_task(FileReconcileService.execute_reconcile_run_services, reconcile_run.run_id) + logger.info(f'文件存储对账任务{reconcile_run.run_id}已启动') + + return ResponseUtil.success(data=reconcile_run, msg='文件存储对账任务已启动') + + +@file_controller.put( + '/reconcile/issues/{issue_id}', + summary='处理文件存储对账异常接口', + description='用于忽略、修复、隔离或登记文件存储异常', + response_model=ResponseBaseModel, + dependencies=[UserInterfaceAuthDependency('system:file:reconcile')], +) +@ApiRateLimit(namespace=ApiNamespace.SYSTEM_FILE_RECONCILE, preset=ApiRateLimitPreset.USER_DESTRUCTIVE_MUTATION) +@Log(title='文件存储对账', business_type=BusinessType.UPDATE) +async def handle_system_file_reconcile_issue( + request: Request, + issue_id: Annotated[int, Path(gt=0, description='对账异常ID')], + handle_reconcile: FileReconcileHandleModel, + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], +) -> Response: + handle_result = await FileReconcileService.handle_reconcile_issue_services( + query_db, + current_user, + issue_id, + handle_reconcile, + request=request, + ) + logger.info(handle_result.message) + + return ResponseUtil.success(msg=handle_result.message) + + +@file_controller.get( + '/retention-policy/list', + summary='获取文件业务保留策略列表接口', + description='用于获取文件业务类型对应的保留策略', + response_model=DataResponseModel[list[FileRetentionPolicyModel]], + dependencies=[UserInterfaceAuthDependency('system:file:list')], +) +async def get_system_file_retention_policy_list( + request: Request, + query_db: Annotated[AsyncSession, DBSessionDependency()], +) -> Response: + policy_list_result = await FileRetentionPolicyService.get_file_retention_policy_list_services(query_db) + logger.info('文件业务保留策略列表获取成功') + + return ResponseUtil.success(data=policy_list_result) + + +@file_controller.post( + '/retention-policy', + summary='新增文件业务保留策略接口', + description='用于新增文件业务类型对应的保留策略', + response_model=ResponseBaseModel, + dependencies=[UserInterfaceAuthDependency('system:file:edit')], +) +@ApiRateLimit( + namespace=ApiNamespace.SYSTEM_FILE_RETENTION_POLICY, + preset=ApiRateLimitPreset.USER_DESTRUCTIVE_MUTATION, +) +@Log(title='文件保留策略', business_type=BusinessType.INSERT) +async def add_system_file_retention_policy( + request: Request, + policy: FileRetentionPolicyModel, + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], +) -> Response: + policy_result = await FileRetentionPolicyService.add_file_retention_policy_services( + query_db, + policy, + current_user.user.user_name, + ) + logger.info(policy_result.message) + + return ResponseUtil.success(msg=policy_result.message) + + +@file_controller.put( + '/retention-policy', + summary='修改文件业务保留策略接口', + description='用于修改文件业务类型对应的保留策略', + response_model=ResponseBaseModel, + dependencies=[UserInterfaceAuthDependency('system:file:edit')], +) +@ApiRateLimit( + namespace=ApiNamespace.SYSTEM_FILE_RETENTION_POLICY, + preset=ApiRateLimitPreset.USER_DESTRUCTIVE_MUTATION, +) +@Log(title='文件保留策略', business_type=BusinessType.UPDATE) +async def edit_system_file_retention_policy( + request: Request, + policy: FileRetentionPolicyModel, + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], +) -> Response: + policy_result = await FileRetentionPolicyService.edit_file_retention_policy_services( + query_db, + policy, + current_user.user.user_name, + ) + logger.info(policy_result.message) + + return ResponseUtil.success(msg=policy_result.message) + + +@file_controller.delete( + '/retention-policy/{business_type}', + summary='删除文件业务保留策略接口', + description='用于删除文件业务类型对应的保留策略', + response_model=ResponseBaseModel, + dependencies=[UserInterfaceAuthDependency('system:file:edit')], +) +@ApiRateLimit( + namespace=ApiNamespace.SYSTEM_FILE_RETENTION_POLICY, + preset=ApiRateLimitPreset.USER_DESTRUCTIVE_MUTATION, +) +@Log(title='文件保留策略', business_type=BusinessType.DELETE) +async def delete_system_file_retention_policy( + request: Request, + business_type: Annotated[str, Path(min_length=1, max_length=50, description='业务类型')], + query_db: Annotated[AsyncSession, DBSessionDependency()], +) -> Response: + policy_result = await FileRetentionPolicyService.delete_file_retention_policy_services( + query_db, + business_type, + ) + logger.info(policy_result.message) + + return ResponseUtil.success(msg=policy_result.message) + + +@file_controller.get( + '/retention-reminder/list', + summary='获取文件保留期限提醒分页列表接口', + description='用于获取当前数据权限范围内的文件保留期限提醒', + response_model=PageResponseModel[FileRetentionNoticeModel], + dependencies=[UserInterfaceAuthDependency('system:file:list')], +) +async def get_system_file_retention_reminder_list( + request: Request, + reminder_page_query: Annotated[FileRetentionNoticePageQueryModel, Query()], + query_db: Annotated[AsyncSession, DBSessionDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], +) -> Response: + reminder_page_query_result = await FileRetentionNoticeService.get_file_retention_notice_list_services( + query_db, + reminder_page_query, + file_data_scope_sql, + is_page=True, + ) + logger.info('文件保留期限提醒获取成功') + + return ResponseUtil.success(model_content=reminder_page_query_result) + + +@file_controller.post( + '/retention-reminder/scan', + summary='执行文件保留期限提醒扫描接口', + description='用于扫描当前数据权限范围内即将到期和已到期的受保护文件', + response_model=DataResponseModel[FileRetentionScanModel], + dependencies=[UserInterfaceAuthDependency('system:file:edit')], +) +@ApiRateLimit( + namespace=ApiNamespace.SYSTEM_FILE_RETENTION_POLICY, + preset=ApiRateLimitPreset.USER_DESTRUCTIVE_MUTATION, +) +@Log(title='文件保留期限提醒', business_type=BusinessType.UPDATE) +async def scan_system_file_retention_reminder( + request: Request, + query_db: Annotated[AsyncSession, DBSessionDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], +) -> Response: + scan_result = await FileRetentionNoticeService.scan_file_retention_notices_services( + query_db, + file_data_scope_sql=file_data_scope_sql, + ) + logger.info( + f'文件保留期限提醒扫描成功，即将到期{scan_result.expiring_count}个，已到期{scan_result.expired_count}个' + ) + + return ResponseUtil.success(data=scan_result) + + +@file_controller.put( + '/retention-reminder/{notice_ids}/read', + summary='标记文件保留期限提醒已读接口', + description='用于将当前数据权限范围内的文件保留期限提醒标记为已读', + response_model=ResponseBaseModel, + dependencies=[UserInterfaceAuthDependency('system:file:list')], +) +@ApiRateLimit( + namespace=ApiNamespace.SYSTEM_FILE_RETENTION_POLICY, + preset=ApiRateLimitPreset.USER_DESTRUCTIVE_MUTATION, +) +async def read_system_file_retention_reminder( + request: Request, + notice_ids: Annotated[str, Path(description='提醒ID')], + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], +) -> Response: + read_result = await FileRetentionNoticeService.mark_file_retention_notices_read_services( + query_db, + notice_ids, + current_user.user.user_name, + file_data_scope_sql, + ) + logger.info(read_result.message) + + return ResponseUtil.success(msg=read_result.message) + + +@file_controller.put( + '/retention-reminder/{notice_id}/extend', + summary='延长文件保留期限接口', + description='用于延长当前数据权限范围内文件的保留期限', + response_model=ResponseBaseModel, + dependencies=[UserInterfaceAuthDependency('system:file:edit')], +) +@ApiRateLimit( + namespace=ApiNamespace.SYSTEM_FILE_RETENTION_POLICY, + preset=ApiRateLimitPreset.USER_DESTRUCTIVE_MUTATION, +) +@Log(title='文件到期处置', business_type=BusinessType.UPDATE) +async def extend_system_file_retention( + request: Request, + notice_id: Annotated[int, Path(gt=0, description='提醒ID')], + extend_retention: ExtendFileRetentionModel, + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], +) -> Response: + extend_result = await FileRetentionDispositionService.extend_file_retention_services( + query_db, + current_user, + notice_id, + extend_retention, + file_data_scope_sql, + request=request, + ) + logger.info(extend_result.message) + + return ResponseUtil.success(msg=extend_result.message) + + +@file_controller.put( + '/retention-reminder/{notice_id}/dispose', + summary='处置到期文件接口', + description='用于释放已到期业务引用并将文件移入回收站', + response_model=ResponseBaseModel, + dependencies=[UserInterfaceAuthDependency('system:file:remove')], +) +@ApiRateLimit(namespace=ApiNamespace.SYSTEM_FILE_DELETE, preset=ApiRateLimitPreset.USER_DESTRUCTIVE_MUTATION) +@Log(title='文件到期处置', business_type=BusinessType.DELETE) +async def dispose_system_expired_file( + request: Request, + notice_id: Annotated[int, Path(gt=0, description='提醒ID')], + dispose_file: DisposeExpiredFileModel, + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], +) -> Response: + dispose_result = await FileRetentionDispositionService.dispose_expired_file_services( + query_db, + current_user, + notice_id, + dispose_file, + file_data_scope_sql, + request=request, + ) + logger.info(dispose_result.message) + + return ResponseUtil.success(msg=dispose_result.message) + + +@file_controller.get( + '/acl/subjects', + summary='查询文件授权主体选项接口', + description='用于按用户、角色或部门查询文件授权主体选项', + response_model=DataResponseModel[list[FileAclSubjectOptionModel]], + dependencies=[UserInterfaceAuthDependency(['system:file:list', 'system:file:edit', 'system:file:transfer'])], +) +async def search_system_file_acl_subjects( + request: Request, + subject_type: Annotated[Literal['user', 'role', 'dept'], Query(alias='subjectType')], + query_db: Annotated[AsyncSession, DBSessionDependency()], + user_data_scope_sql: Annotated[ColumnElement, DataScopeDependency(SysUser)], + dept_data_scope_sql: Annotated[ColumnElement, DataScopeDependency(SysDept)], + keyword: Annotated[str | None, Query(max_length=50)] = None, + limit: Annotated[int, Query(ge=1, le=50)] = 20, +) -> Response: + subject_list_result = await FileAclService.search_file_acl_subjects_services( + query_db, + subject_type, + keyword, + limit, + user_data_scope_sql, + dept_data_scope_sql, + ) + logger.info('文件授权主体查询成功') + + return ResponseUtil.success(data=subject_list_result) + + +@file_controller.get( + '/acl/dept-tree', + summary='获取文件授权部门树接口', + description='用于获取文件授权可选的有效部门树', + response_model=DataResponseModel[list[DeptTreeModel]], + dependencies=[UserInterfaceAuthDependency(['system:file:list', 'system:file:edit', 'system:file:transfer'])], +) +async def get_system_file_acl_dept_tree( + request: Request, + query_db: Annotated[AsyncSession, DBSessionDependency()], + dept_data_scope_sql: Annotated[ColumnElement, DataScopeDependency(SysDept)], +) -> Response: + dept_tree_result = await FileAclService.get_file_acl_dept_tree_services(query_db, dept_data_scope_sql) + logger.info('文件授权部门树获取成功') + + return ResponseUtil.success(data=dept_tree_result) + + +@file_controller.put( + '/acl/batch', + summary='批量保存文件访问控制接口', + description='用于批量替换受保护文件的访问控制配置', + response_model=ResponseBaseModel, + dependencies=[UserInterfaceAuthDependency('system:file:edit')], +) +@ApiRateLimit(namespace=ApiNamespace.SYSTEM_FILE_ACL, preset=ApiRateLimitPreset.USER_DESTRUCTIVE_MUTATION) +@Log(title='文件管理', business_type=BusinessType.UPDATE) +async def batch_save_system_file_acl( + request: Request, + batch_save_file_acl: BatchSaveFileAclModel, + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], + user_data_scope_sql: Annotated[ColumnElement, DataScopeDependency(SysUser)], + dept_data_scope_sql: Annotated[ColumnElement, DataScopeDependency(SysDept)], +) -> Response: + save_file_acl_result = await FileAclService.batch_save_file_acl_services( + query_db, + current_user, + batch_save_file_acl, + file_data_scope_sql, + user_data_scope_sql, + dept_data_scope_sql, + request=request, + ) + logger.info(save_file_acl_result.message) + + return ResponseUtil.success(msg=save_file_acl_result.message) + + +@file_controller.get( + '/{file_id}/acl/list', + summary='获取文件访问控制列表接口', + description='用于获取指定文件的访问控制列表', + response_model=DataResponseModel[FileAclListModel], + dependencies=[UserInterfaceAuthDependency('system:file:edit')], +) +async def get_system_file_acl_list( + request: Request, + file_id: UUID, + query_db: Annotated[AsyncSession, DBSessionDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], + user_data_scope_sql: Annotated[ColumnElement, DataScopeDependency(SysUser)], + dept_data_scope_sql: Annotated[ColumnElement, DataScopeDependency(SysDept)], +) -> Response: + file_acl_list_result = await FileAclService.get_file_acl_list_services( + query_db, + str(file_id), + file_data_scope_sql, + user_data_scope_sql, + dept_data_scope_sql, + ) + logger.info('文件访问控制列表获取成功') + + return ResponseUtil.success(data=file_acl_list_result) + + +@file_controller.put( + '/{file_id}/acl', + summary='保存文件访问控制接口', + description='用于替换指定私有文件的访问控制配置', + response_model=ResponseBaseModel, + dependencies=[UserInterfaceAuthDependency('system:file:edit')], +) +@ApiRateLimit(namespace=ApiNamespace.SYSTEM_FILE_ACL, preset=ApiRateLimitPreset.USER_DESTRUCTIVE_MUTATION) +@Log(title='文件管理', business_type=BusinessType.UPDATE) +async def save_system_file_acl( + request: Request, + file_id: UUID, + save_file_acl: SaveFileAclModel, + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], + user_data_scope_sql: Annotated[ColumnElement, DataScopeDependency(SysUser)], + dept_data_scope_sql: Annotated[ColumnElement, DataScopeDependency(SysDept)], +) -> Response: + save_file_acl_result = await FileAclService.save_file_acl_services( + query_db, + current_user, + str(file_id), + save_file_acl, + file_data_scope_sql, + user_data_scope_sql, + dept_data_scope_sql, + request=request, + ) + logger.info(save_file_acl_result.message) + + return ResponseUtil.success(msg=save_file_acl_result.message) + + +@file_controller.get( + '/{file_id}/reference/list', + summary='获取文件业务引用列表接口', + description='用于获取指定文件的业务引用列表', + response_model=DataResponseModel[list[FileReferenceModel]], + dependencies=[UserInterfaceAuthDependency('system:file:query')], +) +async def get_system_file_reference_list( + request: Request, + file_id: UUID, + query_db: Annotated[AsyncSession, DBSessionDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], +) -> Response: + file_reference_list_result = await FileReferenceService.get_file_reference_list_services( + query_db, + str(file_id), + file_data_scope_sql, + ) + logger.info('文件业务引用列表获取成功') + + return ResponseUtil.success(data=file_reference_list_result) + + +@file_controller.get( + '/{file_id}/access-log/list', + summary='获取文件访问审计分页列表接口', + description='用于获取指定文件的访问审计分页列表', + response_model=PageResponseModel[FileAccessLogModel], + dependencies=[UserInterfaceAuthDependency('system:file:query')], +) +async def get_system_file_access_log_list( + request: Request, + file_id: UUID, + access_log_page_query: Annotated[FileAccessLogPageQueryModel, Query()], + query_db: Annotated[AsyncSession, DBSessionDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], +) -> Response: + access_log_page_query_result = await FileQueryService.get_file_access_log_list_services( + query_db, + str(file_id), + access_log_page_query, + file_data_scope_sql, + is_page=True, + ) + logger.info('获取成功') + + return ResponseUtil.success(model_content=access_log_page_query_result) + + +@file_controller.get( + '/download/{file_id}/{display_name}', + summary='文件管理下载接口', + description='用于具有文件管理权限的用户下载文件', + response_class=StreamingResponse, + responses={ + 200: { + 'description': '流式返回文件', + 'content': { + 'application/octet-stream': {}, + }, + }, + 206: {'description': '分段返回文件'}, + 416: {'description': '请求的字节范围不可满足'}, + }, + dependencies=[UserInterfaceAuthDependency('system:file:download')], +) +@ApiRateLimit(namespace=ApiNamespace.SYSTEM_FILE_DOWNLOAD, preset=ApiRateLimitPreset.USER_RESOURCE_DOWNLOAD) +@Log(title='文件管理', business_type=BusinessType.EXPORT) +async def download_system_file( + request: Request, + file_id: UUID, + display_name: str, + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], +) -> Response: + download_result = await CommonService.download_managed_file_services( + request, + query_db, + current_user, + str(file_id), + enforce_owner_permission=False, + file_data_scope_sql=file_data_scope_sql, + range_header=request.headers.get('Range'), + ) + logger.info(f'文件{file_id}下载成功') + + return ResponseUtil.streaming( + data=download_result.data, + headers=UploadUtil.build_download_headers( + download_result.filename, + download_result.byte_range, + download_result.accept_ranges, + ), + media_type='application/octet-stream', + status_code=206 if download_result.byte_range.is_partial else 200, + ) + + +@file_controller.put( + '/{file_ids}/transfer', + summary='转移文件接口', + description='用于批量转移文件所有者和所属部门', + response_model=ResponseBaseModel, + dependencies=[UserInterfaceAuthDependency('system:file:transfer')], +) +@ApiRateLimit(namespace=ApiNamespace.SYSTEM_FILE_TRANSFER, preset=ApiRateLimitPreset.USER_DESTRUCTIVE_MUTATION) +@Log(title='文件管理', business_type=BusinessType.UPDATE) +async def transfer_system_file( + request: Request, + file_ids: Annotated[str, Path(description='需要转移的文件ID')], + transfer_file: TransferFileModel, + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], + user_data_scope_sql: Annotated[ColumnElement, DataScopeDependency(SysUser)], + dept_data_scope_sql: Annotated[ColumnElement, DataScopeDependency(SysDept)], +) -> Response: + transfer_file_result = await FileTransferService.transfer_file_services( + query_db, + current_user, + file_ids, + transfer_file, + file_data_scope_sql, + user_data_scope_sql, + dept_data_scope_sql, + request=request, + ) + logger.info(transfer_file_result.message) + + return ResponseUtil.success(msg=transfer_file_result.message) + + +@file_controller.delete( + '/{file_ids}', + summary='删除文件接口', + description='用于批量删除文件及其物理内容', + response_model=ResponseBaseModel, + dependencies=[UserInterfaceAuthDependency('system:file:remove')], +) +@ApiRateLimit(namespace=ApiNamespace.SYSTEM_FILE_DELETE, preset=ApiRateLimitPreset.USER_DESTRUCTIVE_MUTATION) +@Log(title='文件管理', business_type=BusinessType.DELETE) +async def delete_system_file( + request: Request, + file_ids: Annotated[str, Path(description='需要删除的文件ID')], + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], +) -> Response: + delete_file_result = await FileLifecycleService.delete_file_services( + query_db, + current_user, + DeleteFileModel(fileIds=file_ids), + file_data_scope_sql, + request=request, + ) + logger.info(delete_file_result.message) + + return ResponseUtil.success(msg=delete_file_result.message) + + +@file_controller.put( + '/{file_ids}/restore', + summary='恢复文件接口', + description='用于批量恢复回收站中的文件', + response_model=ResponseBaseModel, + dependencies=[UserInterfaceAuthDependency('system:file:restore')], +) +@ApiRateLimit(namespace=ApiNamespace.SYSTEM_FILE_RESTORE, preset=ApiRateLimitPreset.USER_DESTRUCTIVE_MUTATION) +@Log(title='文件管理', business_type=BusinessType.UPDATE) +async def restore_system_file( + request: Request, + file_ids: Annotated[str, Path(description='需要恢复的文件ID')], + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], +) -> Response: + restore_file_result = await FileLifecycleService.restore_file_services( + query_db, + current_user, + file_ids, + file_data_scope_sql, + request=request, + ) + logger.info(restore_file_result.message) + + return ResponseUtil.success(msg=restore_file_result.message) + + +@file_controller.delete( + '/purge/{file_ids}', + summary='永久清理回收站文件接口', + description='用于批量永久清理回收站文件及其管理元数据，操作不可恢复', + response_model=ResponseBaseModel, + dependencies=[UserInterfaceAuthDependency('system:file:purge')], +) +@ApiRateLimit(namespace=ApiNamespace.SYSTEM_FILE_DELETE, preset=ApiRateLimitPreset.USER_DESTRUCTIVE_MUTATION) +@Log(title='文件管理', business_type=BusinessType.CLEAN) +async def purge_system_file( + request: Request, + file_ids: Annotated[str, Path(description='需要永久清理的文件ID')], + query_db: Annotated[AsyncSession, DBSessionDependency()], + current_user: Annotated[CurrentUserModel, CurrentUserDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], +) -> Response: + purge_file_result = await FileLifecycleService.purge_file_services( + query_db, + current_user, + file_ids, + file_data_scope_sql, + request=request, + ) + logger.info(purge_file_result.message) + + return ResponseUtil.success(msg=purge_file_result.message) + + +@file_controller.get( + '/{file_id}', + summary='获取文件详情接口', + description='用于获取指定文件的详细信息', + response_model=DataResponseModel[FileInfoDisplayModel], + dependencies=[UserInterfaceAuthDependency('system:file:query')], +) +async def get_system_file_detail( + request: Request, + file_id: UUID, + query_db: Annotated[AsyncSession, DBSessionDependency()], + file_data_scope_sql: Annotated[ + ColumnElement, + DataScopeDependency(SysFileInfo, user_alias='owner_user_id', dept_alias='dept_id'), + ], +) -> Response: + file_detail_result = await FileQueryService.file_detail_services(query_db, str(file_id), file_data_scope_sql) + logger.info(f'获取file_id为{file_id}的信息成功') + + return ResponseUtil.success(data=file_detail_result) diff --git a/ruoyi-fastapi-backend/module_admin/dao/file_access_dao.py b/ruoyi-fastapi-backend/module_admin/dao/file_access_dao.py new file mode 100644 index 0000000..c689ec3 --- /dev/null +++ b/ruoyi-fastapi-backend/module_admin/dao/file_access_dao.py @@ -0,0 +1,366 @@ +from datetime import datetime +from typing import Any + +from sqlalchemy import ColumnElement, delete, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from common.vo import PageModel +from module_admin.entity.do.dept_do import SysDept +from module_admin.entity.do.file_do import SysFileAccessLog, SysFileAcl +from module_admin.entity.do.role_do import SysRole +from module_admin.entity.do.user_do import SysUser, SysUserRole +from module_admin.entity.vo.file_vo import ( + FileAccessLogModel, + FileAccessLogPageQueryModel, +) +from utils.page_util import PageUtil + + +class FileAclDao: + """ + 文件访问控制数据操作层 + """ + + @classmethod + async def get_effective_file_acl_list( + cls, + db: AsyncSession, + file_id: str, + current_time: datetime, + ) -> list[SysFileAcl]: + """ + 获取文件有效访问控制列表 + + :param db: orm对象 + :param file_id: 文件ID + :param current_time: 当前时间 + :return: 文件访问控制列表 + """ + return list( + ( + await db.execute( + select(SysFileAcl).where( + SysFileAcl.file_id == file_id, + SysFileAcl.permission == 'download', + SysFileAcl.del_flag == '0', + or_(SysFileAcl.expire_time.is_(None), SysFileAcl.expire_time > current_time), + ) + ) + ) + .scalars() + .all() + ) + + @classmethod + async def get_file_acl_list(cls, db: AsyncSession, file_id: str) -> list[SysFileAcl]: + """ + 获取文件访问控制列表 + + :param db: orm对象 + :param file_id: 文件ID + :return: 文件访问控制列表 + """ + return list( + ( + await db.execute( + select(SysFileAcl) + .where(SysFileAcl.file_id == file_id, SysFileAcl.del_flag == '0') + .order_by(SysFileAcl.subject_type, SysFileAcl.subject_id, SysFileAcl.acl_id) + ) + ) + .scalars() + .all() + ) + + @classmethod + async def get_acl_dept_list(cls, db: AsyncSession, dept_data_scope_sql: ColumnElement) -> list[SysDept]: + """ + 获取文件授权可选部门列表 + + :param db: orm对象 + :param dept_data_scope_sql: 部门数据权限对应的查询sql语句 + :return: 部门列表 + """ + return list( + ( + await db.execute( + select(SysDept) + .where(SysDept.status == '0', SysDept.del_flag == '0', dept_data_scope_sql) + .order_by(SysDept.order_num, SysDept.dept_id) + ) + ) + .scalars() + .all() + ) + + @classmethod + async def replace_file_acl_list(cls, db: AsyncSession, file_id: str, file_acl_list: list[SysFileAcl]) -> None: + """ + 替换文件访问控制列表 + + :param db: orm对象 + :param file_id: 文件ID + :param file_acl_list: 文件访问控制列表 + :return: None + """ + await db.execute(delete(SysFileAcl).where(SysFileAcl.file_id == file_id)) + if file_acl_list: + db.add_all(file_acl_list) + await db.flush() + + @classmethod + async def replace_file_acl_lists( + cls, + db: AsyncSession, + file_ids: list[str], + file_acl_list: list[SysFileAcl], + ) -> None: + """ + 批量替换文件访问控制列表 + + :param db: orm对象 + :param file_ids: 文件ID列表 + :param file_acl_list: 文件访问控制列表 + :return: None + """ + await db.execute(delete(SysFileAcl).where(SysFileAcl.file_id.in_(file_ids))) + if file_acl_list: + db.add_all(file_acl_list) + await db.flush() + + @classmethod + async def get_acl_subject_name_map( + cls, + db: AsyncSession, + subject_ids: dict[str, set[int]], + user_data_scope_sql: ColumnElement, + dept_data_scope_sql: ColumnElement, + ) -> dict[tuple[str, int], str]: + """ + 获取访问控制主体名称映射 + + :param db: orm对象 + :param subject_ids: 按主体类型分组的主体ID + :param user_data_scope_sql: 用户数据权限对应的查询sql语句 + :param dept_data_scope_sql: 部门数据权限对应的查询sql语句 + :return: 主体名称映射 + """ + subject_name_map = {} + user_ids = subject_ids.get('user', set()) + if user_ids: + user_rows = ( + await db.execute( + select(SysUser.user_id, SysUser.user_name, SysUser.nick_name).where( + SysUser.user_id.in_(user_ids), + SysUser.status == '0', + SysUser.del_flag == '0', + user_data_scope_sql, + ) + ) + ).all() + for user_id, user_name, nick_name in user_rows: + display_name = f'{nick_name}（{user_name}）' if nick_name and nick_name != user_name else user_name + subject_name_map[('user', user_id)] = display_name + + role_ids = subject_ids.get('role', set()) + if role_ids: + role_rows = ( + await db.execute( + select(SysRole.role_id, SysRole.role_name).where( + SysRole.role_id != 1, + SysRole.role_id.in_(role_ids), + SysRole.status == '0', + SysRole.del_flag == '0', + ) + ) + ).all() + role_rows = await cls._filter_role_rows_by_data_scope(db, role_rows, user_data_scope_sql) + subject_name_map.update({('role', role_id): role_name for role_id, role_name in role_rows}) + + dept_ids = subject_ids.get('dept', set()) + if dept_ids: + dept_rows = ( + await db.execute( + select(SysDept.dept_id, SysDept.dept_name).where( + SysDept.dept_id.in_(dept_ids), + SysDept.status == '0', + SysDept.del_flag == '0', + dept_data_scope_sql, + ) + ) + ).all() + subject_name_map.update({('dept', dept_id): dept_name for dept_id, dept_name in dept_rows}) + + return subject_name_map + + @classmethod + async def search_acl_subjects( + cls, + db: AsyncSession, + subject_type: str, + keyword: str | None, + limit: int, + user_data_scope_sql: ColumnElement, + dept_data_scope_sql: ColumnElement, + ) -> list[dict[str, Any]]: + """ + 查询访问控制主体选项 + + :param db: orm对象 + :param subject_type: 主体类型 + :param keyword: 查询关键字 + :param limit: 返回数量限制 + :param user_data_scope_sql: 用户数据权限对应的查询sql语句 + :param dept_data_scope_sql: 部门数据权限对应的查询sql语句 + :return: 主体选项列表 + """ + keyword_pattern = f'%{keyword}%' + if subject_type == 'user': + query = select(SysUser.user_id, SysUser.user_name, SysUser.nick_name, SysUser.dept_id).where( + SysUser.status == '0', + SysUser.del_flag == '0', + user_data_scope_sql, + or_(SysUser.user_name.like(keyword_pattern), SysUser.nick_name.like(keyword_pattern)) + if keyword + else True, + ) + rows = (await db.execute(query.order_by(SysUser.user_id).limit(limit))).all() + return [ + { + 'subject_id': user_id, + 'subject_name': f'{nick_name}（{user_name}）' + if nick_name and nick_name != user_name + else user_name, + 'dept_id': dept_id, + } + for user_id, user_name, nick_name, dept_id in rows + ] + if subject_type == 'role': + query = select(SysRole.role_id, SysRole.role_name).where( + SysRole.role_id != 1, + SysRole.status == '0', + SysRole.del_flag == '0', + SysRole.role_name.like(keyword_pattern) if keyword else True, + ) + rows = (await db.execute(query.order_by(SysRole.role_id))).all() + rows = await cls._filter_role_rows_by_data_scope(db, rows, user_data_scope_sql) + rows = rows[:limit] + return [{'subject_id': role_id, 'subject_name': role_name} for role_id, role_name in rows] + + query = select(SysDept.dept_id, SysDept.dept_name).where( + SysDept.status == '0', + SysDept.del_flag == '0', + dept_data_scope_sql, + SysDept.dept_name.like(keyword_pattern) if keyword else True, + ) + rows = (await db.execute(query.order_by(SysDept.order_num, SysDept.dept_id).limit(limit))).all() + return [{'subject_id': dept_id, 'subject_name': dept_name} for dept_id, dept_name in rows] + + @classmethod + async def _filter_role_rows_by_data_scope( + cls, + db: AsyncSession, + role_rows: list[Any], + user_data_scope_sql: ColumnElement, + ) -> list[Any]: + """ + 过滤包含数据权限范围外成员的角色 + + :param db: orm对象 + :param role_rows: 候选角色列表 + :param user_data_scope_sql: 用户数据权限对应的查询sql语句 + :return: 数据权限范围内的角色列表 + """ + role_ids = {role_id for role_id, _ in role_rows} + if not role_ids: + return [] + + all_member_rows = ( + await db.execute( + select(SysUserRole.role_id, SysUserRole.user_id) + .join(SysUser, SysUser.user_id == SysUserRole.user_id) + .where( + SysUserRole.role_id.in_(role_ids), + SysUser.del_flag == '0', + ) + ) + ).all() + visible_member_rows = ( + await db.execute( + select(SysUserRole.role_id, SysUserRole.user_id) + .join(SysUser, SysUser.user_id == SysUserRole.user_id) + .where( + SysUserRole.role_id.in_(role_ids), + SysUser.del_flag == '0', + user_data_scope_sql, + ) + ) + ).all() + + all_member_ids = {} + for role_id, user_id in all_member_rows: + all_member_ids.setdefault(role_id, set()).add(user_id) + visible_member_ids = {} + for role_id, user_id in visible_member_rows: + visible_member_ids.setdefault(role_id, set()).add(user_id) + + return [ + role_row + for role_row in role_rows + if all_member_ids.get(role_row[0], set()).issubset(visible_member_ids.get(role_row[0], set())) + ] + + +class FileAccessLogDao: + """ + 文件访问审计数据操作层 + """ + + @classmethod + async def add_file_access_log_dao(cls, db: AsyncSession, file_access_log: FileAccessLogModel) -> SysFileAccessLog: + """ + 新增文件访问审计记录 + + :param db: orm对象 + :param file_access_log: 文件访问审计对象 + :return: 文件访问审计数据库对象 + """ + db_file_access_log = SysFileAccessLog(**file_access_log.model_dump(exclude={'audit_id'})) + db.add(db_file_access_log) + await db.flush() + return db_file_access_log + + @classmethod + async def get_file_access_log_list( + cls, + db: AsyncSession, + file_id: str, + query_object: FileAccessLogPageQueryModel, + is_page: bool = False, + ) -> PageModel | list[dict[str, Any]]: + """ + 根据查询参数获取文件访问审计列表 + + :param db: orm对象 + :param file_id: 文件ID + :param query_object: 文件访问审计查询参数 + :param is_page: 是否开启分页 + :return: 文件访问审计列表 + """ + query = ( + select(SysFileAccessLog) + .where( + SysFileAccessLog.file_id == file_id, + SysFileAccessLog.action == query_object.action if query_object.action else True, + SysFileAccessLog.result == query_object.result if query_object.result else True, + SysFileAccessLog.actor_name.like(f'%{query_object.actor_name}%') if query_object.actor_name else True, + SysFileAccessLog.access_time.between( + datetime.strptime(query_object.begin_time, '%Y-%m-%d %H:%M:%S'), + datetime.strptime(query_object.end_time, '%Y-%m-%d %H:%M:%S'), + ) + if query_object.begin_time and query_object.end_time + else True, + ) + .order_by(SysFileAccessLog.access_time.desc(), SysFileAccessLog.audit_id.desc()) + ) + return await PageUtil.paginate(db, query, query_object.page_num, query_object.page_size, is_page) diff --git a/ruoyi-fastapi-backend/module_admin/dao/file_business_dao.py b/ruoyi-fastapi-backend/module_admin/dao/file_business_dao.py new file mode 100644 index 0000000..188e095 --- /dev/null +++ b/ruoyi-fastapi-backend/module_admin/dao/file_business_dao.py @@ -0,0 +1,645 @@ +from datetime import datetime +from typing import Any, Literal + +from sqlalchemy import ColumnElement, and_, delete, exists, func, or_, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from common.vo import PageModel +from module_admin.entity.do.dept_do import SysDept +from module_admin.entity.do.file_do import ( + SysFileInfo, + SysFileReference, + SysFileRetentionNotice, + SysFileRetentionPolicy, +) +from module_admin.entity.do.user_do import SysUser +from module_admin.entity.vo.file_vo import FileRetentionNoticePageQueryModel +from utils.page_util import PageUtil + + +class FileReferenceDao: + """ + 文件业务引用数据操作层 + """ + + @classmethod + async def get_file_reference_list(cls, db: AsyncSession, file_id: str) -> list[SysFileReference]: + """ + 获取文件业务引用列表 + + :param db: orm对象 + :param file_id: 文件ID + :return: 文件业务引用列表 + """ + return list( + ( + await db.execute( + select(SysFileReference) + .where(SysFileReference.file_id == file_id) + .order_by( + SysFileReference.business_type, + SysFileReference.business_id, + SysFileReference.reference_id, + ) + ) + ) + .scalars() + .all() + ) + + @classmethod + async def get_file_reference_list_for_update(cls, db: AsyncSession, file_id: str) -> list[SysFileReference]: + """ + 锁定文件业务引用列表 + + :param db: orm对象 + :param file_id: 文件ID + :return: 文件业务引用列表 + """ + return list( + ( + await db.execute( + select(SysFileReference) + .where(SysFileReference.file_id == file_id) + .order_by(SysFileReference.reference_id) + .with_for_update() + ) + ) + .scalars() + .all() + ) + + @classmethod + async def delete_file_references(cls, db: AsyncSession, file_id: str) -> None: + """ + 删除文件的全部业务引用 + + :param db: orm对象 + :param file_id: 文件ID + :return: None + """ + await db.execute(delete(SysFileReference).where(SysFileReference.file_id == file_id)) + + @classmethod + async def get_file_reference_count_map(cls, db: AsyncSession, file_ids: list[str]) -> dict[str, int]: + """ + 获取文件业务引用数量映射 + + :param db: orm对象 + :param file_ids: 文件ID列表 + :return: 文件ID和业务引用数量映射 + """ + if not file_ids: + return {} + rows = ( + ( + await db.execute( + select( + SysFileReference.file_id, + func.count(SysFileReference.reference_id).label('reference_count'), + ) + .where(SysFileReference.file_id.in_(file_ids)) + .group_by(SysFileReference.file_id) + ) + ) + .mappings() + .all() + ) + return {str(row['file_id']): int(row['reference_count']) for row in rows} + + @classmethod + async def replace_business_file_references( + cls, + db: AsyncSession, + business_type: str, + business_id: str, + file_reference_list: list[SysFileReference], + ) -> None: + """ + 替换业务对象的文件引用 + + :param db: orm对象 + :param business_type: 业务类型 + :param business_id: 业务ID + :param file_reference_list: 文件业务引用列表 + :return: None + """ + old_reference_list = list( + ( + await db.execute( + select(SysFileReference).where( + SysFileReference.business_type == business_type, + SysFileReference.business_id == business_id, + ) + ) + ) + .scalars() + .all() + ) + old_file_ids = [reference.file_id for reference in old_reference_list] + cls._preserve_later_retention_expire_times(old_reference_list, file_reference_list) + affected_file_ids = sorted(set(old_file_ids).union(reference.file_id for reference in file_reference_list)) + file_info_map = {} + if affected_file_ids: + file_info_list = list( + ( + await db.execute( + select(SysFileInfo) + .where(SysFileInfo.file_id.in_(affected_file_ids)) + .order_by(SysFileInfo.file_id) + .with_for_update() + ) + ) + .scalars() + .all() + ) + file_info_map = {file_info.file_id: file_info for file_info in file_info_list} + + await db.execute( + delete(SysFileReference).where( + SysFileReference.business_type == business_type, + SysFileReference.business_id == business_id, + ) + ) + await db.execute( + update(SysFileInfo) + .where( + SysFileInfo.business_type == business_type, + SysFileInfo.business_id == business_id, + ) + .values(business_type=None, business_id=None) + ) + if file_reference_list: + db.add_all(file_reference_list) + await db.flush() + await cls._refresh_file_expire_times(db, affected_file_ids, file_info_map) + + @staticmethod + def _preserve_later_retention_expire_times( + old_reference_list: list[SysFileReference], + new_reference_list: list[SysFileReference], + ) -> None: + """保留同一业务引用已经延长的更晚到期时间。""" + old_reference_map = {reference.file_id: reference for reference in old_reference_list} + for reference in new_reference_list: + old_reference = old_reference_map.get(reference.file_id) + if ( + old_reference + and old_reference.retention_expire_time + and reference.retention_expire_time + and old_reference.retention_expire_time > reference.retention_expire_time + ): + reference.retention_expire_time = old_reference.retention_expire_time + + @classmethod + async def _refresh_file_expire_times( + cls, + db: AsyncSession, + file_ids: list[str], + file_info_map: dict[str, SysFileInfo], + ) -> None: + """根据业务引用重新计算文件过期时间。""" + if not file_ids: + return + rows = ( + await db.execute( + select( + SysFileReference.file_id, + SysFileReference.retention_expire_time, + ).where(SysFileReference.file_id.in_(file_ids)) + ) + ).all() + retention_map: dict[str, list] = {file_id: [] for file_id in file_ids} + for file_id, retention_expire_time in rows: + retention_map[str(file_id)].append(retention_expire_time) + + for file_id in file_ids: + file_info = file_info_map.get(file_id) + if file_info is None: + continue + retention_expire_times = retention_map[file_id] + has_legacy_reference = bool(file_info.business_type and file_info.business_id) + if has_legacy_reference or any(expire_time is None for expire_time in retention_expire_times): + file_info.expire_time = None + else: + file_info.expire_time = max(retention_expire_times, default=None) + await FileRetentionNoticeDao.invalidate_changed_expire_time_notices( + db, + file_id, + file_info.expire_time, + ) + + +class FileRetentionPolicyDao: + """ + 文件业务保留策略数据操作层 + """ + + @classmethod + async def get_file_retention_policy_list(cls, db: AsyncSession) -> list[SysFileRetentionPolicy]: + """ + 获取文件业务保留策略列表 + + :param db: orm对象 + :return: 文件业务保留策略列表 + """ + return list( + (await db.execute(select(SysFileRetentionPolicy).order_by(SysFileRetentionPolicy.business_type))) + .scalars() + .all() + ) + + @classmethod + async def get_file_retention_policy_by_business_type( + cls, + db: AsyncSession, + business_type: str, + enabled_only: bool = False, + ) -> SysFileRetentionPolicy | None: + """ + 根据业务类型获取文件业务保留策略 + + :param db: orm对象 + :param business_type: 业务类型 + :param enabled_only: 是否只查询启用策略 + :return: 文件业务保留策略 + """ + return ( + ( + await db.execute( + select(SysFileRetentionPolicy).where( + SysFileRetentionPolicy.business_type == business_type, + SysFileRetentionPolicy.status == '0' if enabled_only else True, + ) + ) + ) + .scalars() + .first() + ) + + @classmethod + async def add_file_retention_policy( + cls, + db: AsyncSession, + policy: SysFileRetentionPolicy, + ) -> None: + """ + 新增文件业务保留策略 + + :param db: orm对象 + :param policy: 文件业务保留策略 + :return: None + """ + db.add(policy) + await db.flush() + + @classmethod + async def edit_file_retention_policy( + cls, + db: AsyncSession, + business_type: str, + policy: dict, + ) -> None: + """ + 修改文件业务保留策略 + + :param db: orm对象 + :param business_type: 业务类型 + :param policy: 文件业务保留策略 + :return: None + """ + await db.execute( + update(SysFileRetentionPolicy).where(SysFileRetentionPolicy.business_type == business_type).values(**policy) + ) + + @classmethod + async def delete_file_retention_policy(cls, db: AsyncSession, business_type: str) -> None: + """ + 删除文件业务保留策略 + + :param db: orm对象 + :param business_type: 业务类型 + :return: None + """ + await db.execute(delete(SysFileRetentionPolicy).where(SysFileRetentionPolicy.business_type == business_type)) + + +class FileRetentionNoticeDao: + """ + 文件保留期限提醒数据操作层 + """ + + @classmethod + async def get_missing_notice_candidates( + cls, + db: AsyncSession, + notice_type: Literal['expiring', 'expired'], + current_time: datetime, + reminder_deadline: datetime, + batch_size: int, + file_data_scope_sql: ColumnElement | bool = True, + ) -> list[SysFileInfo]: + """ + 获取尚未生成当前到期提醒的文件 + + :param db: orm对象 + :param notice_type: 提醒类型 + :param current_time: 当前时间 + :param reminder_deadline: 提醒截止时间 + :param batch_size: 单批处理数量 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :return: 文件信息数据库对象列表 + """ + expire_condition = ( + SysFileInfo.expire_time <= current_time + if notice_type == 'expired' + else and_( + SysFileInfo.expire_time > current_time, + SysFileInfo.expire_time <= reminder_deadline, + ) + ) + current_notice_exists = exists( + select(SysFileRetentionNotice.notice_id).where( + SysFileRetentionNotice.file_id == SysFileInfo.file_id, + SysFileRetentionNotice.notice_type == notice_type, + SysFileRetentionNotice.expire_time == SysFileInfo.expire_time, + SysFileRetentionNotice.status.in_(['0', '1']), + ) + ) + return list( + ( + await db.execute( + select(SysFileInfo) + .where( + SysFileInfo.access_type == 'private', + SysFileInfo.status == 'active', + SysFileInfo.del_flag == '0', + SysFileInfo.expire_time.is_not(None), + expire_condition, + ~current_notice_exists, + file_data_scope_sql, + ) + .order_by(SysFileInfo.expire_time, SysFileInfo.file_id) + .limit(batch_size) + .with_for_update() + ) + ) + .scalars() + .all() + ) + + @classmethod + async def add_file_retention_notices( + cls, + db: AsyncSession, + notice_list: list[SysFileRetentionNotice], + ) -> None: + """ + 批量新增文件保留期限提醒 + + :param db: orm对象 + :param notice_list: 提醒对象列表 + :return: None + """ + if notice_list: + db.add_all(notice_list) + await db.flush() + + @classmethod + async def invalidate_expiring_notices(cls, db: AsyncSession, file_ids: list[str]) -> None: + """ + 将已经到期文件的即将到期提醒标记为失效 + + :param db: orm对象 + :param file_ids: 文件ID列表 + :return: None + """ + if not file_ids: + return + await db.execute( + update(SysFileRetentionNotice) + .where( + SysFileRetentionNotice.file_id.in_(file_ids), + SysFileRetentionNotice.notice_type == 'expiring', + SysFileRetentionNotice.status.in_(['0', '1']), + ) + .values(status='2') + ) + + @classmethod + async def invalidate_changed_expire_time_notices( + cls, + db: AsyncSession, + file_id: str, + expire_time: datetime | None, + ) -> None: + """ + 将与文件当前到期时间不一致的未读提醒标记为失效 + + :param db: orm对象 + :param file_id: 文件ID + :param expire_time: 文件当前到期时间 + :return: None + """ + expire_condition = ( + True + if expire_time is None + else or_( + SysFileRetentionNotice.expire_time != expire_time, + SysFileRetentionNotice.expire_time.is_(None), + ) + ) + await db.execute( + update(SysFileRetentionNotice) + .where( + SysFileRetentionNotice.file_id == file_id, + SysFileRetentionNotice.status.in_(['0', '1']), + expire_condition, + ) + .values(status='2') + ) + + @classmethod + async def get_file_retention_notice_list( + cls, + db: AsyncSession, + query_object: FileRetentionNoticePageQueryModel, + file_data_scope_sql: ColumnElement, + is_page: bool = False, + ) -> PageModel | list[dict[str, Any]]: + """ + 获取文件保留期限提醒列表 + + :param db: orm对象 + :param query_object: 查询参数 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :param is_page: 是否开启分页 + :return: 文件保留期限提醒列表 + """ + current_time = datetime.now() + reference_count = ( + select(func.count(SysFileReference.reference_id)) + .where(SysFileReference.file_id == SysFileInfo.file_id) + .correlate(SysFileInfo) + .scalar_subquery() + ) + blocking_reference_exists = exists( + select(SysFileReference.reference_id).where( + SysFileReference.file_id == SysFileInfo.file_id, + or_( + SysFileReference.retention_expire_time.is_(None), + SysFileReference.retention_expire_time > current_time, + ), + ) + ) + query = ( + select( + *SysFileRetentionNotice.__table__.c, + SysFileInfo.original_name, + SysUser.user_name.label('owner_name'), + SysDept.dept_name.label('dept_name'), + reference_count.label('reference_count'), + and_( + SysFileInfo.expire_time <= current_time, + SysFileInfo.business_type.is_(None), + SysFileInfo.business_id.is_(None), + ~blocking_reference_exists, + ).label('can_dispose'), + ) + .join(SysFileInfo, SysFileInfo.file_id == SysFileRetentionNotice.file_id) + .outerjoin(SysUser, SysUser.user_id == SysFileInfo.owner_user_id) + .outerjoin(SysDept, SysDept.dept_id == SysFileInfo.dept_id) + .where( + SysFileInfo.access_type == 'private', + SysFileInfo.status == 'active', + SysFileInfo.del_flag == '0', + SysFileInfo.expire_time == SysFileRetentionNotice.expire_time, + SysFileRetentionNotice.status.in_(['0', '1']), + file_data_scope_sql, + SysFileInfo.original_name.like(f'%{query_object.original_name}%') + if query_object.original_name + else True, + SysFileRetentionNotice.notice_type == query_object.notice_type if query_object.notice_type else True, + SysFileRetentionNotice.status == query_object.status if query_object.status else True, + ) + .order_by( + SysFileRetentionNotice.status, + SysFileRetentionNotice.expire_time, + SysFileRetentionNotice.notice_id.desc(), + ) + ) + return await PageUtil.paginate(db, query, query_object.page_num, query_object.page_size, is_page) + + @classmethod + async def get_file_retention_notice_context_for_update( + cls, + db: AsyncSession, + notice_id: int, + file_data_scope_sql: ColumnElement, + ) -> tuple[SysFileRetentionNotice, SysFileInfo] | None: + """ + 锁定数据权限范围内的有效提醒和文件 + + :param db: orm对象 + :param notice_id: 提醒ID + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :return: 提醒和文件数据库对象 + """ + row = ( + await db.execute( + select(SysFileRetentionNotice, SysFileInfo) + .join(SysFileInfo, SysFileInfo.file_id == SysFileRetentionNotice.file_id) + .where( + SysFileRetentionNotice.notice_id == notice_id, + SysFileRetentionNotice.status.in_(['0', '1']), + SysFileInfo.access_type == 'private', + SysFileInfo.status == 'active', + SysFileInfo.del_flag == '0', + SysFileInfo.expire_time == SysFileRetentionNotice.expire_time, + file_data_scope_sql, + ) + .with_for_update() + ) + ).first() + return (row[0], row[1]) if row else None + + @classmethod + async def get_notice_ids_in_data_scope_for_update( + cls, + db: AsyncSession, + notice_ids: list[int], + file_data_scope_sql: ColumnElement, + ) -> list[int]: + """ + 锁定数据权限范围内的有效提醒 + + :param db: orm对象 + :param notice_ids: 提醒ID列表 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :return: 提醒ID列表 + """ + return list( + ( + await db.execute( + select(SysFileRetentionNotice.notice_id) + .join(SysFileInfo, SysFileInfo.file_id == SysFileRetentionNotice.file_id) + .where( + SysFileRetentionNotice.notice_id.in_(notice_ids), + SysFileRetentionNotice.status.in_(['0', '1']), + SysFileInfo.access_type == 'private', + SysFileInfo.status == 'active', + SysFileInfo.del_flag == '0', + SysFileInfo.expire_time == SysFileRetentionNotice.expire_time, + file_data_scope_sql, + ) + .order_by(SysFileRetentionNotice.notice_id) + .with_for_update() + ) + ) + .scalars() + .all() + ) + + @classmethod + async def mark_file_retention_notices_read( + cls, + db: AsyncSession, + notice_ids: list[int], + read_by: str, + read_time: datetime, + ) -> None: + """ + 标记文件保留期限提醒为已读 + + :param db: orm对象 + :param notice_ids: 提醒ID列表 + :param read_by: 读取者 + :param read_time: 读取时间 + :return: None + """ + await db.execute( + update(SysFileRetentionNotice) + .where( + SysFileRetentionNotice.notice_id.in_(notice_ids), + SysFileRetentionNotice.status == '0', + ) + .values(status='1', read_by=read_by, read_time=read_time) + ) + + @classmethod + async def invalidate_file_retention_notices(cls, db: AsyncSession, file_id: str) -> None: + """ + 将文件当前有效的保留期限提醒标记为失效 + + :param db: orm对象 + :param file_id: 文件ID + :return: None + """ + await db.execute( + update(SysFileRetentionNotice) + .where( + SysFileRetentionNotice.file_id == file_id, + SysFileRetentionNotice.status.in_(['0', '1']), + ) + .values(status='2') + ) diff --git a/ruoyi-fastapi-backend/module_admin/dao/file_info_dao.py b/ruoyi-fastapi-backend/module_admin/dao/file_info_dao.py new file mode 100644 index 0000000..6159174 --- /dev/null +++ b/ruoyi-fastapi-backend/module_admin/dao/file_info_dao.py @@ -0,0 +1,1209 @@ +from datetime import datetime, timedelta +from typing import Any + +from sqlalchemy import ColumnElement, case, delete, exists, func, or_, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from common.vo import PageModel +from module_admin.entity.do.dept_do import SysDept +from module_admin.entity.do.file_do import ( + SysFileAcl, + SysFileInfo, + SysFileReconcileIssue, + SysFileReconcileRun, + SysFileReference, + SysFileRetentionNotice, +) +from module_admin.entity.do.user_do import SysUser +from module_admin.entity.vo.file_vo import ( + FileInfoModel, + FileInfoPageQueryModel, + FileReconcileIssuePageQueryModel, + FileReconcileRunPageQueryModel, + FileStatsModel, +) +from utils.page_util import PageUtil + + +class FileInfoDao: + """ + 文件信息数据操作层 + """ + + FILE_EXPIRING_DAYS = 7 + ACL_EXPIRING_DAYS = 7 + + @classmethod + async def add_file_info_dao(cls, db: AsyncSession, file_info: FileInfoModel) -> SysFileInfo: + """ + 新增文件信息 + + :param db: orm对象 + :param file_info: 文件信息对象 + :return: 文件信息数据库对象 + """ + db_file_info = SysFileInfo(**file_info.model_dump()) + db.add(db_file_info) + await db.flush() + return db_file_info + + @classmethod + async def get_file_info_by_id( + cls, + db: AsyncSession, + file_id: str, + file_data_scope_sql: ColumnElement | None = None, + ) -> SysFileInfo | None: + """ + 根据文件ID获取有效文件信息 + + :param db: orm对象 + :param file_id: 文件ID + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :return: 文件信息数据库对象 + """ + return ( + ( + await db.execute( + select(SysFileInfo).where( + SysFileInfo.file_id == file_id, + SysFileInfo.status == 'active', + SysFileInfo.del_flag == '0', + file_data_scope_sql if file_data_scope_sql is not None else True, + ) + ) + ) + .scalars() + .first() + ) + + @classmethod + async def release_stale_runs(cls, db: AsyncSession, stale_before: datetime, current_time: datetime) -> None: + """ + 释放超时未完成的对账任务锁 + + :param db: orm对象 + :param stale_before: 超时边界 + :param current_time: 当前时间 + :return: None + """ + await db.execute( + update(SysFileReconcileRun) + .where( + SysFileReconcileRun.status == 'running', + SysFileReconcileRun.started_time < stale_before, + ) + .values( + status='failed', + lock_name=None, + finished_time=current_time, + error_message='对账任务运行超时，已自动释放运行锁', + ) + ) + + @classmethod + async def add_reconcile_run(cls, db: AsyncSession, reconcile_run: SysFileReconcileRun) -> None: + """ + 新增文件存储对账任务 + + :param db: orm对象 + :param reconcile_run: 对账任务 + :return: None + """ + db.add(reconcile_run) + await db.flush() + + @classmethod + async def get_reconcile_run_by_id(cls, db: AsyncSession, run_id: str) -> SysFileReconcileRun | None: + """ + 根据任务ID获取文件存储对账任务 + + :param db: orm对象 + :param run_id: 任务ID + :return: 对账任务 + """ + return ( + (await db.execute(select(SysFileReconcileRun).where(SysFileReconcileRun.run_id == run_id))) + .scalars() + .first() + ) + + @classmethod + async def has_running_reconcile_run(cls, db: AsyncSession) -> bool: + """ + 判断是否存在运行中的文件存储对账任务 + + :param db: orm对象 + :return: 是否正在运行 + """ + return bool(await db.scalar(select(exists().where(SysFileReconcileRun.status == 'running')))) + + @classmethod + async def get_reconcile_run_list( + cls, + db: AsyncSession, + query_object: FileReconcileRunPageQueryModel, + is_page: bool = True, + ) -> PageModel | list[dict[str, Any]]: + """ + 获取文件存储对账任务列表 + + :param db: orm对象 + :param query_object: 查询参数 + :param is_page: 是否分页 + :return: 对账任务列表 + """ + query = ( + select( + SysFileReconcileRun.run_id, + SysFileReconcileRun.trigger_type, + SysFileReconcileRun.status, + SysFileReconcileRun.check_hash, + SysFileReconcileRun.scanned_file_count, + SysFileReconcileRun.scanned_storage_count, + SysFileReconcileRun.issue_count, + SysFileReconcileRun.new_issue_count, + SysFileReconcileRun.resolved_issue_count, + SysFileReconcileRun.started_by, + SysFileReconcileRun.started_time, + SysFileReconcileRun.finished_time, + SysFileReconcileRun.error_message, + ) + .where( + SysFileReconcileRun.status == query_object.status if query_object.status else True, + SysFileReconcileRun.trigger_type == query_object.trigger_type if query_object.trigger_type else True, + ) + .order_by(SysFileReconcileRun.started_time.desc(), SysFileReconcileRun.run_id.desc()) + ) + return await PageUtil.paginate(db, query, query_object.page_num, query_object.page_size, is_page) + + @classmethod + async def get_all_local_file_infos(cls, db: AsyncSession) -> list[dict[str, Any]]: + """ + 获取全部本地文件存储信息 + + :param db: orm对象 + :return: 文件存储信息列表 + """ + rows = ( + ( + await db.execute( + select( + SysFileInfo.file_id, + SysFileInfo.storage_type, + SysFileInfo.access_type, + SysFileInfo.storage_key, + SysFileInfo.stored_name, + SysFileInfo.file_size, + SysFileInfo.file_hash, + SysFileInfo.status, + SysFileInfo.del_flag, + ).where(SysFileInfo.storage_type == 'local') + ) + ) + .mappings() + .all() + ) + return [dict(row) for row in rows] + + @classmethod + async def upsert_reconcile_issues( + cls, + db: AsyncSession, + run_id: str, + findings: list[dict[str, Any]], + current_time: datetime, + ) -> int: + """ + 新增或更新文件存储对账异常 + + :param db: orm对象 + :param run_id: 任务ID + :param findings: 对账异常列表 + :param current_time: 当前时间 + :return: 新增或重新出现异常数 + """ + issue_keys = [finding['issue_key'] for finding in findings] + issue_map: dict[str, SysFileReconcileIssue] = {} + for start in range(0, len(issue_keys), 500): + batch_keys = issue_keys[start : start + 500] + issue_map.update( + { + issue.issue_key: issue + for issue in ( + ( + await db.execute( + select(SysFileReconcileIssue).where(SysFileReconcileIssue.issue_key.in_(batch_keys)) + ) + ) + .scalars() + .all() + ) + } + ) + + new_issue_count = 0 + for finding in findings: + issue = issue_map.get(finding['issue_key']) + if issue is None: + db.add( + SysFileReconcileIssue( + **finding, + last_run_id=run_id, + status='open', + occurrence_count=1, + first_seen_time=current_time, + last_seen_time=current_time, + ) + ) + new_issue_count += 1 + continue + if issue.status in {'resolved', 'quarantined'}: + issue.status = 'open' + issue.handle_action = 'reopened_by_scan' + issue.handle_reason = '异常在后续扫描中再次出现' + issue.handled_by = 'system' + issue.handled_time = current_time + new_issue_count += 1 + for field_name, value in finding.items(): + if field_name != 'issue_key': + setattr(issue, field_name, value) + issue.last_run_id = run_id + issue.last_seen_time = current_time + issue.occurrence_count = (issue.occurrence_count or 0) + 1 + await db.flush() + return new_issue_count + + @classmethod + async def resolve_disappeared_issues( + cls, + db: AsyncSession, + run_id: str, + current_time: datetime, + ) -> int: + """ + 自动关闭本次扫描未再次出现的异常 + + :param db: orm对象 + :param run_id: 任务ID + :param current_time: 当前时间 + :return: 自动关闭数量 + """ + result = await db.execute( + update(SysFileReconcileIssue) + .where( + SysFileReconcileIssue.status.in_(['open', 'ignored']), + SysFileReconcileIssue.last_run_id != run_id, + ) + .values( + status='resolved', + handle_action='auto_resolved', + handle_reason='异常在后续完整扫描中未再次出现', + handled_by='system', + handled_time=current_time, + ) + ) + return int(result.rowcount or 0) + + @classmethod + async def finish_reconcile_run( + cls, + db: AsyncSession, + run_id: str, + *, + status: str, + finished_time: datetime, + scanned_file_count: int = 0, + scanned_storage_count: int = 0, + issue_count: int = 0, + new_issue_count: int = 0, + resolved_issue_count: int = 0, + error_message: str = '', + ) -> None: + """ + 完成文件存储对账任务 + + :return: None + """ + await db.execute( + update(SysFileReconcileRun) + .where(SysFileReconcileRun.run_id == run_id) + .values( + status=status, + lock_name=None, + finished_time=finished_time, + scanned_file_count=scanned_file_count, + scanned_storage_count=scanned_storage_count, + issue_count=issue_count, + new_issue_count=new_issue_count, + resolved_issue_count=resolved_issue_count, + error_message=error_message, + ) + ) + + @classmethod + async def get_reconcile_issue_list( + cls, + db: AsyncSession, + query_object: FileReconcileIssuePageQueryModel, + is_page: bool = True, + ) -> PageModel | list[dict[str, Any]]: + """ + 获取文件存储对账异常列表 + + :param db: orm对象 + :param query_object: 查询参数 + :param is_page: 是否分页 + :return: 对账异常列表 + """ + keyword_condition: ColumnElement | bool = True + if query_object.keyword: + keyword = f'%{query_object.keyword}%' + keyword_condition = or_( + SysFileReconcileIssue.file_id.like(keyword), + SysFileInfo.original_name.like(keyword), + SysFileReconcileIssue.expected_key.like(keyword), + SysFileReconcileIssue.actual_key.like(keyword), + ) + query = ( + select( + *SysFileReconcileIssue.__table__.c, + SysFileInfo.original_name, + ) + .outerjoin(SysFileInfo, SysFileInfo.file_id == SysFileReconcileIssue.file_id) + .where( + SysFileReconcileIssue.issue_type == query_object.issue_type if query_object.issue_type else True, + SysFileReconcileIssue.severity == query_object.severity if query_object.severity else True, + SysFileReconcileIssue.status == query_object.status if query_object.status else True, + keyword_condition, + ) + .order_by( + case( + (SysFileReconcileIssue.status == 'open', 0), + (SysFileReconcileIssue.status == 'quarantined', 1), + (SysFileReconcileIssue.status == 'ignored', 2), + else_=3, + ), + case((SysFileReconcileIssue.severity == 'critical', 0), else_=1), + SysFileReconcileIssue.last_seen_time.desc(), + SysFileReconcileIssue.issue_id.desc(), + ) + ) + return await PageUtil.paginate(db, query, query_object.page_num, query_object.page_size, is_page) + + @classmethod + async def get_reconcile_issue_for_update( + cls, + db: AsyncSession, + issue_id: int, + ) -> SysFileReconcileIssue | None: + """ + 锁定文件存储对账异常 + + :param db: orm对象 + :param issue_id: 异常ID + :return: 对账异常 + """ + return ( + ( + await db.execute( + select(SysFileReconcileIssue).where(SysFileReconcileIssue.issue_id == issue_id).with_for_update() + ) + ) + .scalars() + .first() + ) + + @classmethod + async def get_file_info_for_reconcile( + cls, + db: AsyncSession, + file_id: str, + ) -> SysFileInfo | None: + """ + 锁定对账异常关联的文件信息 + + :param db: orm对象 + :param file_id: 文件ID + :return: 文件信息 + """ + return ( + (await db.execute(select(SysFileInfo).where(SysFileInfo.file_id == file_id).with_for_update())) + .scalars() + .first() + ) + + @classmethod + async def resolve_file_integrity_issues( + cls, + db: AsyncSession, + file_id: str, + current_time: datetime, + handled_by: str, + reason: str, + ) -> None: + """ + 关闭文件大小和摘要不一致异常 + + :return: None + """ + await db.execute( + update(SysFileReconcileIssue) + .where( + SysFileReconcileIssue.file_id == file_id, + SysFileReconcileIssue.issue_type.in_(['size_mismatch', 'hash_mismatch']), + SysFileReconcileIssue.status.in_(['open', 'ignored']), + ) + .values( + status='resolved', + handle_action='accept_current', + handle_reason=reason, + handled_by=handled_by, + handled_time=current_time, + ) + ) + + @classmethod + async def get_reconcile_stats(cls, db: AsyncSession) -> dict[str, Any]: + """ + 获取文件存储对账统计 + + :param db: orm对象 + :return: 对账统计 + """ + row = ( + ( + await db.execute( + select( + func.coalesce( + func.sum(case((SysFileReconcileIssue.status == 'open', 1), else_=0)), + 0, + ).label('open_count'), + func.coalesce( + func.sum( + case( + ( + (SysFileReconcileIssue.status == 'open') + & (SysFileReconcileIssue.severity == 'critical'), + 1, + ), + else_=0, + ) + ), + 0, + ).label('critical_count'), + func.coalesce( + func.sum( + case( + ( + (SysFileReconcileIssue.status == 'open') + & (SysFileReconcileIssue.severity == 'warning'), + 1, + ), + else_=0, + ) + ), + 0, + ).label('warning_count'), + func.coalesce( + func.sum(case((SysFileReconcileIssue.status == 'ignored', 1), else_=0)), + 0, + ).label('ignored_count'), + func.coalesce( + func.sum(case((SysFileReconcileIssue.status == 'quarantined', 1), else_=0)), + 0, + ).label('quarantined_count'), + ) + ) + ) + .mappings() + .one() + ) + latest_run = ( + (await db.execute(select(SysFileReconcileRun).order_by(SysFileReconcileRun.started_time.desc()).limit(1))) + .scalars() + .first() + ) + return {**dict(row), 'latest_run': latest_run} + + @classmethod + async def get_file_info_detail_by_id( + cls, + db: AsyncSession, + file_id: str, + file_data_scope_sql: ColumnElement | None = None, + ) -> SysFileInfo | None: + """ + 根据文件ID获取文件详细信息 + + :param db: orm对象 + :param file_id: 文件ID + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :return: 文件信息数据库对象 + """ + return ( + ( + await db.execute( + select(SysFileInfo).where( + SysFileInfo.file_id == file_id, + file_data_scope_sql if file_data_scope_sql is not None else True, + ) + ) + ) + .scalars() + .first() + ) + + @classmethod + async def get_file_info_by_id_for_update( + cls, + db: AsyncSession, + file_id: str, + file_data_scope_sql: ColumnElement | None = None, + ) -> SysFileInfo | None: + """ + 根据文件ID锁定有效文件信息 + + :param db: orm对象 + :param file_id: 文件ID + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :return: 文件信息数据库对象 + """ + return ( + ( + await db.execute( + select(SysFileInfo) + .where( + SysFileInfo.file_id == file_id, + SysFileInfo.status == 'active', + SysFileInfo.del_flag == '0', + file_data_scope_sql if file_data_scope_sql is not None else True, + ) + .with_for_update() + ) + ) + .scalars() + .first() + ) + + @classmethod + async def get_file_infos_by_ids_for_update( + cls, + db: AsyncSession, + file_ids: list[str], + file_data_scope_sql: ColumnElement | None = None, + ) -> list[SysFileInfo]: + """ + 根据文件ID列表锁定有效文件信息 + + :param db: orm对象 + :param file_ids: 文件ID列表 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :return: 文件信息数据库对象列表 + """ + return list( + ( + await db.execute( + select(SysFileInfo) + .where( + SysFileInfo.file_id.in_(file_ids), + SysFileInfo.status == 'active', + SysFileInfo.del_flag == '0', + file_data_scope_sql if file_data_scope_sql is not None else True, + ) + .order_by(SysFileInfo.file_id) + .with_for_update() + ) + ) + .scalars() + .all() + ) + + @classmethod + async def get_deleted_file_infos_by_ids_for_update( + cls, + db: AsyncSession, + file_ids: list[str], + file_data_scope_sql: ColumnElement | None = None, + ) -> list[SysFileInfo]: + """ + 根据文件ID列表锁定已删除文件信息 + + :param db: orm对象 + :param file_ids: 文件ID列表 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :return: 文件信息数据库对象列表 + """ + return list( + ( + await db.execute( + select(SysFileInfo) + .where( + SysFileInfo.file_id.in_(file_ids), + SysFileInfo.status == 'deleted', + SysFileInfo.del_flag == '1', + file_data_scope_sql if file_data_scope_sql is not None else True, + ) + .order_by(SysFileInfo.file_id) + .with_for_update() + ) + ) + .scalars() + .all() + ) + + @classmethod + async def get_purgeable_file_infos_by_ids_for_update( + cls, + db: AsyncSession, + file_ids: list[str], + file_data_scope_sql: ColumnElement | None = None, + ) -> list[SysFileInfo]: + """ + 根据文件ID列表锁定可永久清理的文件信息 + + :param db: orm对象 + :param file_ids: 文件ID列表 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :return: 文件信息数据库对象列表 + """ + return list( + ( + await db.execute( + select(SysFileInfo) + .where( + SysFileInfo.file_id.in_(file_ids), + SysFileInfo.status.in_(['deleted', 'purging']), + SysFileInfo.del_flag == '1', + file_data_scope_sql if file_data_scope_sql is not None else True, + ) + .order_by(SysFileInfo.file_id) + .with_for_update() + ) + ) + .scalars() + .all() + ) + + @classmethod + async def get_recycle_bin_purge_candidates( + cls, + db: AsyncSession, + deleted_before: datetime, + batch_size: int, + ) -> list[SysFileInfo]: + """ + 获取自动清理的回收站文件 + + :param db: orm对象 + :param deleted_before: 最晚删除时间 + :param batch_size: 单批处理数量 + :return: 文件信息数据库对象列表 + """ + return list( + ( + await db.execute( + select(SysFileInfo) + .where( + SysFileInfo.del_flag == '1', + or_( + SysFileInfo.status == 'purging', + (SysFileInfo.status == 'deleted') & (SysFileInfo.deleted_time <= deleted_before), + ), + or_(SysFileInfo.business_type.is_(None), SysFileInfo.business_id.is_(None)), + ~exists( + select(SysFileReference.reference_id).where(SysFileReference.file_id == SysFileInfo.file_id) + ), + ) + .order_by( + case((SysFileInfo.status == 'purging', 0), else_=1), + SysFileInfo.deleted_time, + SysFileInfo.file_id, + ) + .limit(batch_size) + .with_for_update() + ) + ) + .scalars() + .all() + ) + + @classmethod + async def get_file_info_list( + cls, + db: AsyncSession, + query_object: FileInfoPageQueryModel, + file_data_scope_sql: ColumnElement, + is_page: bool = False, + ) -> PageModel | list[dict[str, Any]]: + """ + 根据查询参数获取文件信息列表 + + :param db: orm对象 + :param query_object: 文件信息查询参数 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :param is_page: 是否开启分页 + :return: 文件信息列表 + """ + current_time = datetime.now() + acl_summary = ( + select( + SysFileAcl.file_id.label('acl_file_id'), + func.min(SysFileAcl.expire_time).label('acl_nearest_expire_time'), + func.count(SysFileAcl.acl_id).label('acl_entry_count'), + ) + .where( + SysFileAcl.del_flag == '0', + or_(SysFileAcl.expire_time.is_(None), SysFileAcl.expire_time > current_time), + ) + .group_by(SysFileAcl.file_id) + .subquery() + ) + query = ( + select( + *SysFileInfo.__table__.c, + SysUser.user_name.label('owner_name'), + SysDept.dept_name.label('dept_name'), + acl_summary.c.acl_nearest_expire_time, + func.coalesce(acl_summary.c.acl_entry_count, 0).label('acl_entry_count'), + ) + .outerjoin(SysUser, SysUser.user_id == SysFileInfo.owner_user_id) + .outerjoin(SysDept, SysDept.dept_id == SysFileInfo.dept_id) + .outerjoin(acl_summary, acl_summary.c.acl_file_id == SysFileInfo.file_id) + .where( + file_data_scope_sql, + *cls._get_file_info_query_conditions(query_object, current_time), + ) + .order_by(SysFileInfo.create_time.desc(), SysFileInfo.file_id.desc()) + ) + return await PageUtil.paginate(db, query, query_object.page_num, query_object.page_size, is_page) + + @classmethod + async def get_file_management_detail_by_id( + cls, + db: AsyncSession, + file_id: str, + file_data_scope_sql: ColumnElement, + ) -> dict[str, Any] | None: + """ + 获取包含管理展示字段的文件详情 + + :param db: orm对象 + :param file_id: 文件ID + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :return: 文件管理详情 + """ + current_time = datetime.now() + acl_summary = ( + select( + SysFileAcl.file_id.label('acl_file_id'), + func.min(SysFileAcl.expire_time).label('acl_nearest_expire_time'), + func.count(SysFileAcl.acl_id).label('acl_entry_count'), + ) + .where( + SysFileAcl.del_flag == '0', + or_(SysFileAcl.expire_time.is_(None), SysFileAcl.expire_time > current_time), + ) + .group_by(SysFileAcl.file_id) + .subquery() + ) + row = ( + ( + await db.execute( + select( + *SysFileInfo.__table__.c, + SysUser.user_name.label('owner_name'), + SysDept.dept_name.label('dept_name'), + acl_summary.c.acl_nearest_expire_time, + func.coalesce(acl_summary.c.acl_entry_count, 0).label('acl_entry_count'), + ) + .outerjoin(SysUser, SysUser.user_id == SysFileInfo.owner_user_id) + .outerjoin(SysDept, SysDept.dept_id == SysFileInfo.dept_id) + .outerjoin(acl_summary, acl_summary.c.acl_file_id == SysFileInfo.file_id) + .where(SysFileInfo.file_id == file_id, file_data_scope_sql) + ) + ) + .mappings() + .first() + ) + return dict(row) if row else None + + @classmethod + async def get_file_stats( + cls, + db: AsyncSession, + query_object: FileInfoPageQueryModel, + file_data_scope_sql: ColumnElement, + ) -> FileStatsModel: + """ + 获取文件管理统计信息 + + :param db: orm对象 + :param query_object: 文件信息查询参数 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :return: 文件管理统计信息 + """ + current_time = datetime.now() + acl_expiring_time = current_time + timedelta(days=cls.ACL_EXPIRING_DAYS) + acl_expiring_files = ( + select(SysFileAcl.file_id) + .where( + SysFileAcl.del_flag == '0', + SysFileAcl.expire_time > current_time, + SysFileAcl.expire_time <= acl_expiring_time, + ) + .distinct() + .subquery() + ) + row = ( + ( + await db.execute( + select( + func.count(SysFileInfo.file_id).label('total_count'), + func.coalesce(func.sum(SysFileInfo.file_size), 0).label('total_size'), + func.coalesce( + func.sum(case((SysFileInfo.access_type == 'public', SysFileInfo.file_size), else_=0)), + 0, + ).label('public_size'), + func.coalesce( + func.sum(case((SysFileInfo.access_type == 'private', SysFileInfo.file_size), else_=0)), + 0, + ).label('private_size'), + func.coalesce(func.sum(case((SysFileInfo.status == 'active', 1), else_=0)), 0).label( + 'active_count' + ), + func.coalesce( + func.sum(case((SysFileInfo.status.in_(['deleted', 'purging']), 1), else_=0)), + 0, + ).label('deleted_count'), + func.coalesce( + func.sum( + case( + ( + (SysFileInfo.access_type == 'private') + & (SysFileInfo.status == 'active') + & (SysFileInfo.expire_time.is_not(None)) + & (SysFileInfo.expire_time <= current_time), + 1, + ), + else_=0, + ) + ), + 0, + ).label('expired_count'), + func.coalesce( + func.sum( + case( + ( + (SysFileInfo.access_type == 'private') + & (SysFileInfo.status == 'active') + & (SysFileInfo.expire_time > current_time) + & ( + SysFileInfo.expire_time + <= current_time + timedelta(days=cls.FILE_EXPIRING_DAYS) + ), + 1, + ), + else_=0, + ) + ), + 0, + ).label('retention_expiring_count'), + func.count(acl_expiring_files.c.file_id).label('acl_expiring_count'), + ) + .outerjoin(SysUser, SysUser.user_id == SysFileInfo.owner_user_id) + .outerjoin(acl_expiring_files, acl_expiring_files.c.file_id == SysFileInfo.file_id) + .where( + file_data_scope_sql, + *cls._get_file_info_query_conditions(query_object, current_time), + ) + ) + ) + .mappings() + .one() + ) + return FileStatsModel.model_validate(dict(row), by_name=True) + + @classmethod + def _get_file_info_query_conditions( + cls, + query_object: FileInfoPageQueryModel, + current_time: datetime, + ) -> list[ColumnElement | bool]: + """ + 构建文件管理查询条件 + + :param query_object: 文件信息查询参数 + :param current_time: 当前时间 + :return: 查询条件列表 + """ + expiring_time = current_time + timedelta(days=cls.FILE_EXPIRING_DAYS) + expiration_condition: ColumnElement | bool = True + if query_object.expiration_status == 'permanent': + expiration_condition = SysFileInfo.expire_time.is_(None) + elif query_object.expiration_status == 'expired': + expiration_condition = SysFileInfo.expire_time <= current_time + elif query_object.expiration_status == 'expiring': + expiration_condition = (SysFileInfo.expire_time > current_time) & (SysFileInfo.expire_time <= expiring_time) + elif query_object.expiration_status == 'valid': + expiration_condition = SysFileInfo.expire_time > expiring_time + return [ + SysFileInfo.original_name.like(f'%{query_object.original_name}%') if query_object.original_name else True, + SysFileInfo.access_type == query_object.access_type if query_object.access_type else True, + SysFileInfo.status == query_object.status if query_object.status else True, + SysFileInfo.create_by.like(f'%{query_object.create_by}%') if query_object.create_by else True, + or_( + SysUser.user_name.like(f'%{query_object.owner_name}%'), + SysUser.nick_name.like(f'%{query_object.owner_name}%'), + ) + if query_object.owner_name + else True, + SysFileInfo.dept_id == query_object.dept_id if query_object.dept_id else True, + expiration_condition, + SysFileInfo.create_time.between( + datetime.strptime(query_object.begin_time, '%Y-%m-%d %H:%M:%S'), + datetime.strptime(query_object.end_time, '%Y-%m-%d %H:%M:%S'), + ) + if query_object.begin_time and query_object.end_time + else True, + ] + + @classmethod + async def soft_delete_file_infos( + cls, + db: AsyncSession, + file_ids: list[str], + update_by: str, + update_time: datetime, + ) -> None: + """ + 逻辑删除文件信息 + + :param db: orm对象 + :param file_ids: 文件ID列表 + :param update_by: 更新者 + :param update_time: 更新时间 + :return: None + """ + await db.execute( + update(SysFileInfo) + .where(SysFileInfo.file_id.in_(file_ids)) + .values( + status='deleted', + del_flag='1', + update_by=update_by, + update_time=update_time, + deleted_time=update_time, + ) + ) + + @classmethod + async def restore_file_infos( + cls, + db: AsyncSession, + file_ids: list[str], + update_by: str, + update_time: datetime, + ) -> None: + """ + 恢复文件信息 + + :param db: orm对象 + :param file_ids: 文件ID列表 + :param update_by: 更新者 + :param update_time: 更新时间 + :return: None + """ + await db.execute( + update(SysFileInfo) + .where(SysFileInfo.file_id.in_(file_ids)) + .values( + status='active', + del_flag='0', + update_by=update_by, + update_time=update_time, + deleted_time=None, + ) + ) + + @classmethod + async def mark_file_infos_purging( + cls, + db: AsyncSession, + file_ids: list[str], + update_by: str, + update_time: datetime, + ) -> None: + """ + 标记文件正在执行永久清理 + + :param db: orm对象 + :param file_ids: 文件ID列表 + :param update_by: 更新者 + :param update_time: 更新时间 + :return: None + """ + await db.execute( + update(SysFileInfo) + .where( + SysFileInfo.file_id.in_(file_ids), + SysFileInfo.status.in_(['deleted', 'purging']), + SysFileInfo.del_flag == '1', + ) + .values( + status='purging', + update_by=update_by, + update_time=update_time, + ) + ) + + @classmethod + async def purge_file_infos(cls, db: AsyncSession, file_ids: list[str]) -> None: + """ + 永久删除文件元数据及关联管理数据 + + :param db: orm对象 + :param file_ids: 文件ID列表 + :return: None + """ + await db.execute(delete(SysFileAcl).where(SysFileAcl.file_id.in_(file_ids))) + await db.execute(delete(SysFileReference).where(SysFileReference.file_id.in_(file_ids))) + await db.execute(delete(SysFileRetentionNotice).where(SysFileRetentionNotice.file_id.in_(file_ids))) + await db.execute( + delete(SysFileInfo).where( + SysFileInfo.file_id.in_(file_ids), + SysFileInfo.status == 'purging', + SysFileInfo.del_flag == '1', + ) + ) + + @classmethod + async def get_transfer_user_by_id( + cls, + db: AsyncSession, + user_id: int, + user_data_scope_sql: ColumnElement, + ) -> SysUser | None: + """ + 获取数据权限范围内的文件转移目标用户 + + :param db: orm对象 + :param user_id: 用户ID + :param user_data_scope_sql: 用户数据权限对应的查询sql语句 + :return: 用户信息 + """ + return ( + ( + await db.execute( + select(SysUser) + .where( + SysUser.user_id == user_id, + SysUser.status == '0', + SysUser.del_flag == '0', + user_data_scope_sql, + ) + .with_for_update() + ) + ) + .scalars() + .first() + ) + + @classmethod + async def get_transfer_dept_by_id( + cls, + db: AsyncSession, + dept_id: int, + dept_data_scope_sql: ColumnElement, + ) -> SysDept | None: + """ + 获取数据权限范围内的文件转移目标部门 + + :param db: orm对象 + :param dept_id: 部门ID + :param dept_data_scope_sql: 部门数据权限对应的查询sql语句 + :return: 部门信息 + """ + return ( + ( + await db.execute( + select(SysDept) + .where( + SysDept.dept_id == dept_id, + SysDept.status == '0', + SysDept.del_flag == '0', + dept_data_scope_sql, + ) + .with_for_update() + ) + ) + .scalars() + .first() + ) + + @classmethod + async def transfer_file_infos( + cls, + db: AsyncSession, + file_ids: list[str], + owner_user_id: int, + dept_id: int, + retain_uploader_access: bool, + update_by: str, + update_time: datetime, + ) -> None: + """ + 批量转移文件所有者和所属部门 + + :param db: orm对象 + :param file_ids: 文件ID列表 + :param owner_user_id: 新所有者用户ID + :param dept_id: 新所属部门ID + :param retain_uploader_access: 是否保留上传人访问权限 + :param update_by: 更新者 + :param update_time: 更新时间 + :return: None + """ + await db.execute( + update(SysFileInfo) + .where(SysFileInfo.file_id.in_(file_ids)) + .values( + owner_user_id=owner_user_id, + dept_id=dept_id, + uploader_access_enabled='1' if retain_uploader_access else '0', + acl_version=SysFileInfo.acl_version + 1, + update_by=update_by, + update_time=update_time, + ) + ) + + @classmethod + async def get_file_info_by_storage_key( + cls, + db: AsyncSession, + storage_key: str, + access_type: str = 'public', + storage_type: str = 'local', + ) -> SysFileInfo | None: + """ + 根据存储相对路径获取文件信息 + + :param db: orm对象 + :param storage_key: 存储相对路径 + :param access_type: 文件访问类型 + :param storage_type: 文件存储类型 + :return: 文件信息数据库对象 + """ + return ( + ( + await db.execute( + select(SysFileInfo).where( + SysFileInfo.storage_key == storage_key, + SysFileInfo.access_type == access_type, + SysFileInfo.storage_type == storage_type, + ) + ) + ) + .scalars() + .first() + ) diff --git a/ruoyi-fastapi-backend/module_admin/entity/do/file_do.py b/ruoyi-fastapi-backend/module_admin/entity/do/file_do.py new file mode 100644 index 0000000..9a90341 --- /dev/null +++ b/ruoyi-fastapi-backend/module_admin/entity/do/file_do.py @@ -0,0 +1,261 @@ +from datetime import datetime + +from sqlalchemy import CHAR, BigInteger, Column, DateTime, Index, Integer, String, Text, UniqueConstraint + +from config.database import Base + + +class SysFileInfo(Base): + """ + 文件信息表 + """ + + __tablename__ = 'sys_file_info' + __table_args__ = ( + UniqueConstraint( + 'storage_type', + 'access_type', + 'storage_key', + name='uk_sys_file_info_storage_location', + ), + Index('idx_sys_file_info_access_status', 'access_type', 'status'), + Index('idx_sys_file_info_owner_status', 'owner_user_id', 'status'), + Index('idx_sys_file_info_dept_status', 'dept_id', 'status'), + Index('idx_sys_file_info_status_deleted_time', 'status', 'deleted_time'), + {'comment': '文件信息表'}, + ) + + file_id = Column(String(36), primary_key=True, nullable=False, comment='文件ID') + original_name = Column(String(255), nullable=False, comment='原始文件名') + stored_name = Column(String(255), nullable=False, comment='存储文件名') + storage_key = Column(String(500), nullable=False, comment='存储相对路径') + storage_type = Column(String(20), nullable=False, server_default='local', comment='存储类型') + access_type = Column(String(20), nullable=False, server_default='public', comment='访问类型') + upload_user_id = Column(BigInteger, nullable=True, comment='上传用户ID') + uploader_access_enabled = Column( + CHAR(1), + nullable=False, + server_default='1', + comment='是否保留上传人访问权限', + ) + owner_user_id = Column(BigInteger, nullable=True, comment='所有者用户ID') + dept_id = Column(BigInteger, nullable=True, comment='所属部门ID') + acl_version = Column(Integer, nullable=False, server_default='0', comment='访问控制版本') + business_type = Column(String(50), nullable=True, comment='业务类型') + business_id = Column(String(64), nullable=True, comment='业务ID') + extension = Column(String(20), nullable=False, server_default="''", comment='文件扩展名') + content_type = Column(String(255), nullable=True, comment='内容类型') + file_size = Column(BigInteger, nullable=False, server_default='0', comment='文件大小') + file_hash = Column(String(64), nullable=False, comment='文件SHA-256') + status = Column(String(20), nullable=False, server_default='active', comment='文件状态') + create_by = Column(String(64), nullable=True, server_default="''", comment='创建者') + create_time = Column(DateTime, nullable=False, default=datetime.now, comment='创建时间') + update_by = Column(String(64), nullable=True, server_default="''", comment='更新者') + update_time = Column(DateTime, nullable=False, default=datetime.now, comment='更新时间') + expire_time = Column(DateTime, nullable=True, comment='过期时间') + deleted_time = Column(DateTime, nullable=True, comment='移入回收站时间') + del_flag = Column(CHAR(1), nullable=False, server_default='0', comment='删除标志') + + +class SysFileReference(Base): + """ + 文件业务引用表 + """ + + __tablename__ = 'sys_file_reference' + __table_args__ = ( + UniqueConstraint( + 'file_id', + 'business_type', + 'business_id', + name='uk_sys_file_reference_business', + ), + Index('idx_sys_file_reference_file', 'file_id'), + Index('idx_sys_file_reference_business', 'business_type', 'business_id'), + {'comment': '文件业务引用表'}, + ) + + reference_id = Column(BigInteger, primary_key=True, nullable=False, autoincrement=True, comment='引用ID') + file_id = Column(String(36), nullable=False, comment='文件ID') + business_type = Column(String(50), nullable=False, comment='业务类型') + business_id = Column(String(64), nullable=False, comment='业务ID') + business_name = Column(String(255), nullable=True, comment='业务名称') + retention_expire_time = Column(DateTime, nullable=True, comment='保留期限到期时间') + create_by = Column(String(64), nullable=True, server_default="''", comment='创建者') + create_time = Column(DateTime, nullable=False, default=datetime.now, comment='创建时间') + + +class SysFileRetentionPolicy(Base): + """ + 文件业务保留策略表 + """ + + __tablename__ = 'sys_file_retention_policy' + __table_args__ = {'comment': '文件业务保留策略表'} + + business_type = Column(String(50), primary_key=True, nullable=False, comment='业务类型') + retention_days = Column(Integer, nullable=False, comment='保留天数') + status = Column(CHAR(1), nullable=False, server_default='0', comment='状态（0启用 1停用）') + remark = Column(String(500), nullable=True, comment='备注') + create_by = Column(String(64), nullable=True, server_default="''", comment='创建者') + create_time = Column(DateTime, nullable=False, default=datetime.now, comment='创建时间') + update_by = Column(String(64), nullable=True, server_default="''", comment='更新者') + update_time = Column(DateTime, nullable=False, default=datetime.now, comment='更新时间') + + +class SysFileRetentionNotice(Base): + """ + 文件保留期限提醒表 + """ + + __tablename__ = 'sys_file_retention_notice' + __table_args__ = ( + UniqueConstraint( + 'file_id', + 'notice_type', + 'expire_time', + name='uk_sys_file_retention_notice_file_type_time', + ), + Index('idx_sys_file_retention_notice_file', 'file_id'), + Index('idx_sys_file_retention_notice_status_time', 'status', 'create_time'), + {'comment': '文件保留期限提醒表'}, + ) + + notice_id = Column(BigInteger, primary_key=True, nullable=False, autoincrement=True, comment='提醒ID') + file_id = Column(String(36), nullable=False, comment='文件ID') + notice_type = Column(String(20), nullable=False, comment='提醒类型') + expire_time = Column(DateTime, nullable=False, comment='文件过期时间') + status = Column(CHAR(1), nullable=False, server_default='0', comment='状态（0未读 1已读 2已失效）') + create_time = Column(DateTime, nullable=False, default=datetime.now, comment='创建时间') + read_by = Column(String(64), nullable=True, server_default="''", comment='读取者') + read_time = Column(DateTime, nullable=True, comment='读取时间') + + +class SysFileAcl(Base): + """ + 文件访问控制表 + """ + + __tablename__ = 'sys_file_acl' + __table_args__ = ( + UniqueConstraint( + 'file_id', + 'subject_type', + 'subject_id', + 'permission', + name='uk_sys_file_acl_subject_permission', + ), + Index('idx_sys_file_acl_file_status', 'file_id', 'del_flag', 'expire_time'), + Index('idx_sys_file_acl_subject', 'subject_type', 'subject_id'), + {'comment': '文件访问控制表'}, + ) + + acl_id = Column(BigInteger, primary_key=True, nullable=False, autoincrement=True, comment='访问控制ID') + file_id = Column(String(36), nullable=False, comment='文件ID') + subject_type = Column(String(20), nullable=False, comment='主体类型') + subject_id = Column(BigInteger, nullable=False, comment='主体ID') + permission = Column(String(20), nullable=False, server_default='download', comment='权限类型') + effect = Column(String(10), nullable=False, server_default='allow', comment='授权效果') + include_children = Column(CHAR(1), nullable=False, server_default='0', comment='部门是否包含下级') + expire_time = Column(DateTime, nullable=True, comment='授权过期时间') + create_by = Column(String(64), nullable=True, server_default="''", comment='创建者') + create_time = Column(DateTime, nullable=False, default=datetime.now, comment='创建时间') + del_flag = Column(CHAR(1), nullable=False, server_default='0', comment='删除标志') + + +class SysFileAccessLog(Base): + """ + 文件访问审计表 + """ + + __tablename__ = 'sys_file_access_log' + __table_args__ = ( + Index('idx_sys_file_access_log_file_time', 'file_id', 'access_time'), + Index('idx_sys_file_access_log_actor_time', 'actor_user_id', 'access_time'), + {'comment': '文件访问审计表'}, + ) + + audit_id = Column(BigInteger, primary_key=True, nullable=False, autoincrement=True, comment='审计ID') + file_id = Column(String(36), nullable=False, comment='文件ID') + action = Column(String(20), nullable=False, comment='操作类型') + actor_user_id = Column(BigInteger, nullable=True, comment='操作用户ID') + actor_name = Column(String(64), nullable=True, server_default="''", comment='操作用户名称') + result = Column(String(20), nullable=False, comment='操作结果') + request_id = Column(String(64), nullable=True, server_default="''", comment='请求ID') + trace_id = Column(String(64), nullable=True, server_default="''", comment='链路ID') + ip_address = Column(String(128), nullable=True, server_default="''", comment='客户端地址') + user_agent = Column(String(500), nullable=True, server_default="''", comment='用户代理') + bytes_sent = Column(BigInteger, nullable=False, server_default='0', comment='发送字节数') + error_message = Column(String(500), nullable=True, server_default="''", comment='失败原因') + operation_detail = Column(Text, nullable=True, comment='操作详情') + access_time = Column(DateTime, nullable=False, default=datetime.now, comment='访问时间') + + +class SysFileReconcileRun(Base): + """ + 文件存储对账任务表 + """ + + __tablename__ = 'sys_file_reconcile_run' + __table_args__ = ( + UniqueConstraint('lock_name', name='uk_sys_file_reconcile_run_lock'), + Index('idx_sys_file_reconcile_run_status_time', 'status', 'started_time'), + {'comment': '文件存储对账任务表'}, + ) + + run_id = Column(String(36), primary_key=True, nullable=False, comment='任务ID') + trigger_type = Column(String(20), nullable=False, comment='触发类型') + status = Column(String(20), nullable=False, comment='任务状态') + check_hash = Column(CHAR(1), nullable=False, server_default='0', comment='是否校验文件摘要') + lock_name = Column(String(32), nullable=True, comment='运行锁名称') + scanned_file_count = Column(BigInteger, nullable=False, server_default='0', comment='扫描文件记录数') + scanned_storage_count = Column(BigInteger, nullable=False, server_default='0', comment='扫描物理文件数') + issue_count = Column(BigInteger, nullable=False, server_default='0', comment='发现异常数') + new_issue_count = Column(BigInteger, nullable=False, server_default='0', comment='新增或重新出现异常数') + resolved_issue_count = Column(BigInteger, nullable=False, server_default='0', comment='自动恢复异常数') + started_by = Column(String(64), nullable=True, server_default="''", comment='发起人') + started_time = Column(DateTime, nullable=False, default=datetime.now, comment='开始时间') + finished_time = Column(DateTime, nullable=True, comment='完成时间') + error_message = Column(Text, nullable=True, comment='失败原因') + + +class SysFileReconcileIssue(Base): + """ + 文件存储对账异常表 + """ + + __tablename__ = 'sys_file_reconcile_issue' + __table_args__ = ( + UniqueConstraint('issue_key', name='uk_sys_file_reconcile_issue_key'), + Index('idx_sys_file_reconcile_issue_status_severity', 'status', 'severity'), + Index('idx_sys_file_reconcile_issue_file', 'file_id'), + Index('idx_sys_file_reconcile_issue_run', 'last_run_id'), + {'comment': '文件存储对账异常表'}, + ) + + issue_id = Column(BigInteger, primary_key=True, nullable=False, autoincrement=True, comment='异常ID') + issue_key = Column(String(64), nullable=False, comment='异常唯一标识') + last_run_id = Column(String(36), nullable=False, comment='最近发现任务ID') + issue_type = Column(String(32), nullable=False, comment='异常类型') + severity = Column(String(10), nullable=False, comment='严重级别') + file_id = Column(String(36), nullable=True, comment='文件ID') + storage_type = Column(String(20), nullable=True, comment='存储类型') + access_type = Column(String(20), nullable=True, comment='访问类型') + expected_root = Column(String(20), nullable=True, comment='预期存储区域') + expected_key = Column(String(500), nullable=True, comment='预期相对路径') + actual_root = Column(String(20), nullable=True, comment='实际存储区域') + actual_key = Column(String(500), nullable=True, comment='实际相对路径') + expected_size = Column(BigInteger, nullable=True, comment='预期文件大小') + actual_size = Column(BigInteger, nullable=True, comment='实际文件大小') + expected_hash = Column(String(64), nullable=True, comment='预期SHA-256') + actual_hash = Column(String(64), nullable=True, comment='实际SHA-256') + status = Column(String(20), nullable=False, server_default='open', comment='处理状态') + detail = Column(Text, nullable=True, comment='异常说明') + occurrence_count = Column(Integer, nullable=False, server_default='1', comment='发现次数') + first_seen_time = Column(DateTime, nullable=False, default=datetime.now, comment='首次发现时间') + last_seen_time = Column(DateTime, nullable=False, default=datetime.now, comment='最近发现时间') + handle_action = Column(String(32), nullable=True, comment='处理动作') + handle_reason = Column(String(500), nullable=True, comment='处理原因') + handled_by = Column(String(64), nullable=True, comment='处理人') + handled_time = Column(DateTime, nullable=True, comment='处理时间') + quarantine_key = Column(String(500), nullable=True, comment='隔离区相对路径') diff --git a/ruoyi-fastapi-backend/module_admin/entity/vo/common_vo.py b/ruoyi-fastapi-backend/module_admin/entity/vo/common_vo.py index 8538055..94afbc6 100644 --- a/ruoyi-fastapi-backend/module_admin/entity/vo/common_vo.py +++ b/ruoyi-fastapi-backend/module_admin/entity/vo/common_vo.py @@ -1,3 +1,5 @@ +from typing import Literal + from pydantic import BaseModel, ConfigDict, Field from pydantic.alias_generators import to_camel @@ -13,3 +15,6 @@ class UploadResponseModel(BaseModel): new_file_name: str | None = Field(default=None, description='新文件名称') original_filename: str | None = Field(default=None, description='原文件名称') url: str | None = Field(default=None, description='新文件url') + file_id: str | None = Field(default=None, description='文件ID') + access_type: Literal['public', 'private'] | None = Field(default=None, description='访问类型') + download_url: str | None = Field(default=None, description='鉴权下载地址') diff --git a/ruoyi-fastapi-backend/module_admin/entity/vo/file_vo.py b/ruoyi-fastapi-backend/module_admin/entity/vo/file_vo.py new file mode 100644 index 0000000..d7dca64 --- /dev/null +++ b/ruoyi-fastapi-backend/module_admin/entity/vo/file_vo.py @@ -0,0 +1,551 @@ +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field +from pydantic.alias_generators import to_camel + +FileReconcileAction = Literal[ + 'ignore', + 'reopen', + 'restore_source', + 'move_to_trash', + 'move_to_expected_root', + 'quarantine_file', + 'restore_quarantine', + 'delete_quarantine', + 'accept_current', + 'register_orphan', +] + + +class FileInfoModel(BaseModel): + """ + 文件信息表对应pydantic模型 + """ + + model_config = ConfigDict(alias_generator=to_camel, from_attributes=True) + + file_id: str = Field(max_length=36, description='文件ID') + original_name: str = Field(max_length=255, description='原始文件名') + stored_name: str = Field(max_length=255, description='存储文件名') + storage_key: str = Field(max_length=500, description='存储相对路径') + storage_type: str = Field(default='local', max_length=20, description='存储类型') + access_type: Literal['public', 'private'] = Field(description='访问类型') + upload_user_id: int | None = Field(default=None, description='上传用户ID') + uploader_access_enabled: Literal['0', '1'] = Field(default='1', description='是否保留上传人访问权限') + owner_user_id: int | None = Field(default=None, description='所有者用户ID') + dept_id: int | None = Field(default=None, description='所属部门ID') + acl_version: int = Field(default=0, ge=0, description='访问控制版本') + business_type: str | None = Field(default=None, max_length=50, description='业务类型') + business_id: str | None = Field(default=None, max_length=64, description='业务ID') + extension: str = Field(max_length=20, description='文件扩展名') + content_type: str | None = Field(default=None, max_length=255, description='内容类型') + file_size: int = Field(default=0, ge=0, description='文件大小') + file_hash: str = Field(min_length=64, max_length=64, description='文件SHA-256') + status: Literal['active', 'deleted', 'purging'] = Field(default='active', description='文件状态') + create_by: str | None = Field(default=None, max_length=64, description='创建者') + create_time: datetime | None = Field(default=None, description='创建时间') + update_by: str | None = Field(default=None, max_length=64, description='更新者') + update_time: datetime | None = Field(default=None, description='更新时间') + expire_time: datetime | None = Field(default=None, description='过期时间') + deleted_time: datetime | None = Field(default=None, description='移入回收站时间') + del_flag: Literal['0', '1'] = Field(default='0', description='删除标志') + + +class FileInfoDisplayModel(FileInfoModel): + """ + 文件信息展示模型 + """ + + owner_name: str | None = Field(default=None, description='所有者用户名称') + dept_name: str | None = Field(default=None, description='所属部门名称') + acl_nearest_expire_time: datetime | None = Field(default=None, description='最近ACL过期时间') + acl_entry_count: int = Field(default=0, ge=0, description='ACL配置数量') + reference_count: int = Field(default=0, ge=0, description='业务引用数量') + storage_status: Literal['normal', 'missing', 'quarantined', 'invalid'] = Field( + default='normal', description='物理存储状态' + ) + + +class FileReferenceModel(BaseModel): + """ + 文件业务引用表对应pydantic模型 + """ + + model_config = ConfigDict(alias_generator=to_camel, from_attributes=True) + + reference_id: int | None = Field(default=None, description='引用ID') + file_id: str = Field(description='文件ID') + business_type: str = Field(description='业务类型') + business_id: str = Field(description='业务ID') + business_name: str | None = Field(default=None, description='业务名称') + retention_expire_time: datetime | None = Field(default=None, description='保留期限到期时间') + create_by: str | None = Field(default=None, description='创建者') + create_time: datetime | None = Field(default=None, description='创建时间') + legacy: bool = Field(default=False, description='是否为文件主表兼容引用') + + +class FileRetentionPolicyModel(BaseModel): + """ + 文件业务保留策略模型 + """ + + model_config = ConfigDict(alias_generator=to_camel, from_attributes=True, str_strip_whitespace=True) + + business_type: str = Field(min_length=1, max_length=50, description='业务类型') + retention_days: int = Field(ge=1, le=36500, description='保留天数') + status: Literal['0', '1'] = Field(default='0', description='状态（0启用 1停用）') + remark: str | None = Field(default=None, max_length=500, description='备注') + create_by: str | None = Field(default=None, max_length=64, description='创建者') + create_time: datetime | None = Field(default=None, description='创建时间') + update_by: str | None = Field(default=None, max_length=64, description='更新者') + update_time: datetime | None = Field(default=None, description='更新时间') + + +class FileRetentionNoticeModel(BaseModel): + """ + 文件保留期限提醒展示模型 + """ + + model_config = ConfigDict(alias_generator=to_camel, from_attributes=True) + + notice_id: int = Field(description='提醒ID') + file_id: str = Field(description='文件ID') + original_name: str = Field(description='原始文件名') + owner_name: str | None = Field(default=None, description='所有者用户名称') + dept_name: str | None = Field(default=None, description='所属部门名称') + notice_type: Literal['expiring', 'expired'] = Field(description='提醒类型') + expire_time: datetime = Field(description='文件过期时间') + status: Literal['0', '1'] = Field(description='状态（0未读 1已读）') + create_time: datetime = Field(description='创建时间') + read_by: str | None = Field(default=None, description='读取者') + read_time: datetime | None = Field(default=None, description='读取时间') + reference_count: int = Field(default=0, ge=0, description='业务引用数量') + can_dispose: bool = Field(default=False, description='是否允许到期处置') + + +class FileRetentionNoticeQueryModel(BaseModel): + """ + 文件保留期限提醒不分页查询模型 + """ + + model_config = ConfigDict(alias_generator=to_camel) + + original_name: str | None = Field(default=None, description='原始文件名') + notice_type: Literal['expiring', 'expired'] | None = Field(default=None, description='提醒类型') + status: Literal['0', '1'] | None = Field(default=None, description='提醒状态') + + +class FileRetentionNoticePageQueryModel(FileRetentionNoticeQueryModel): + """ + 文件保留期限提醒分页查询模型 + """ + + page_num: int = Field(default=1, description='当前页码') + page_size: int = Field(default=10, description='每页记录数') + + +class FileRetentionScanModel(BaseModel): + """ + 文件保留期限提醒扫描结果模型 + """ + + model_config = ConfigDict(alias_generator=to_camel) + + expiring_count: int = Field(default=0, ge=0, description='新增即将到期提醒数') + expired_count: int = Field(default=0, ge=0, description='新增已到期提醒数') + + +class ExtendFileRetentionModel(BaseModel): + """ + 延长文件保留期限模型 + """ + + model_config = ConfigDict(alias_generator=to_camel, str_strip_whitespace=True) + + expire_time: datetime = Field(description='新的到期时间') + reason: str = Field(min_length=1, max_length=500, description='延期原因') + + +class DisposeExpiredFileModel(BaseModel): + """ + 到期文件处置模型 + """ + + model_config = ConfigDict(alias_generator=to_camel, str_strip_whitespace=True) + + reason: str = Field(min_length=1, max_length=500, description='处置原因') + + +class FileAccessLogModel(BaseModel): + """ + 文件访问审计表对应pydantic模型 + """ + + model_config = ConfigDict(alias_generator=to_camel, from_attributes=True) + + audit_id: int | None = Field(default=None, description='审计ID') + file_id: str = Field(description='文件ID') + action: Literal[ + 'upload', + 'download', + 'acl_update', + 'transfer', + 'delete', + 'restore', + 'purge', + 'reconcile', + 'retention_extend', + 'retention_dispose', + ] = Field(description='操作类型') + actor_user_id: int | None = Field(default=None, description='操作用户ID') + actor_name: str | None = Field(default=None, description='操作用户名称') + result: Literal['allowed', 'denied', 'completed', 'failed'] = Field(description='操作结果') + request_id: str | None = Field(default=None, description='请求ID') + trace_id: str | None = Field(default=None, description='链路ID') + ip_address: str | None = Field(default=None, description='客户端地址') + user_agent: str | None = Field(default=None, description='用户代理') + bytes_sent: int = Field(default=0, description='发送字节数') + error_message: str | None = Field(default=None, description='失败原因') + operation_detail: str | None = Field(default=None, description='操作详情') + access_time: datetime | None = Field(default=None, description='访问时间') + + +class FileAclModel(BaseModel): + """ + 文件访问控制表对应pydantic模型 + """ + + model_config = ConfigDict(alias_generator=to_camel, from_attributes=True) + + acl_id: int | None = Field(default=None, description='访问控制ID') + file_id: str = Field(description='文件ID') + subject_type: Literal['user', 'role', 'dept'] = Field(description='主体类型') + subject_id: int = Field(description='主体ID') + subject_name: str | None = Field(default=None, description='主体名称') + permission: Literal['download'] = Field(default='download', description='权限类型') + effect: Literal['allow', 'deny'] = Field(description='授权效果') + include_children: bool = Field(default=False, description='部门是否包含下级') + expire_time: datetime | None = Field(default=None, description='授权过期时间') + create_by: str | None = Field(default=None, description='创建者') + create_time: datetime | None = Field(default=None, description='创建时间') + + +class FileAclItemModel(BaseModel): + """ + 文件访问控制配置项模型 + """ + + model_config = ConfigDict(alias_generator=to_camel) + + subject_type: Literal['user', 'role', 'dept'] = Field(description='主体类型') + subject_id: int = Field(gt=0, description='主体ID') + effect: Literal['allow', 'deny'] = Field(description='授权效果') + include_children: bool = Field(default=False, description='部门是否包含下级') + expire_time: datetime | None = Field(default=None, description='授权过期时间') + + +class FileAclBuiltinPermissionModel(BaseModel): + """ + 文件内置访问权限展示模型 + """ + + model_config = ConfigDict(alias_generator=to_camel) + + source: Literal['admin', 'owner', 'uploader'] = Field(description='权限来源') + subject_id: int | None = Field(default=None, description='权限主体ID') + subject_name: str = Field(description='权限主体名称') + permission: Literal['download'] = Field(default='download', description='权限类型') + enabled: bool = Field(description='权限是否启用') + deny_overridable: bool = Field(description='是否可以被显式拒绝覆盖') + description: str = Field(description='权限说明') + + +class FileAclListModel(BaseModel): + """ + 文件访问控制列表模型 + """ + + model_config = ConfigDict(alias_generator=to_camel) + + acl_version: int = Field(ge=0, description='访问控制版本') + builtin_permissions: list[FileAclBuiltinPermissionModel] = Field( + default_factory=list, description='文件内置访问权限' + ) + entries: list[FileAclModel] = Field(default_factory=list, description='访问控制配置项') + + +class SaveFileAclModel(BaseModel): + """ + 保存文件访问控制模型 + """ + + model_config = ConfigDict(alias_generator=to_camel) + + acl_version: int = Field(ge=0, description='访问控制版本') + entries: list[FileAclItemModel] = Field(default_factory=list, max_length=100, description='访问控制配置项') + + +class BatchSaveFileAclModel(BaseModel): + """ + 批量保存文件访问控制模型 + """ + + model_config = ConfigDict(alias_generator=to_camel) + + file_ids: str = Field(description='需要授权的文件ID') + entries: list[FileAclItemModel] = Field(default_factory=list, max_length=100, description='访问控制配置项') + + +class FileAclSubjectOptionModel(BaseModel): + """ + 文件访问控制主体选项模型 + """ + + model_config = ConfigDict(alias_generator=to_camel) + + subject_id: int = Field(description='主体ID') + subject_name: str = Field(description='主体名称') + dept_id: int | None = Field(default=None, description='所属部门ID') + + +class FileInfoQueryModel(BaseModel): + """ + 文件信息管理不分页查询模型 + """ + + model_config = ConfigDict(alias_generator=to_camel) + + original_name: str | None = Field(default=None, description='原始文件名') + access_type: Literal['public', 'private'] | None = Field(default=None, description='访问类型') + status: Literal['active', 'deleted', 'purging'] | None = Field(default=None, description='文件状态') + create_by: str | None = Field(default=None, description='上传用户名称') + owner_name: str | None = Field(default=None, description='所有者用户名称') + dept_id: int | None = Field(default=None, gt=0, description='所属部门ID') + expiration_status: Literal['permanent', 'valid', 'expiring', 'expired'] | None = Field( + default=None, description='文件过期状态' + ) + begin_time: str | None = Field(default=None, description='开始时间') + end_time: str | None = Field(default=None, description='结束时间') + + +class FileInfoPageQueryModel(FileInfoQueryModel): + """ + 文件信息管理分页查询模型 + """ + + page_num: int = Field(default=1, description='当前页码') + page_size: int = Field(default=10, description='每页记录数') + + +class FileStatsModel(BaseModel): + """ + 文件管理统计模型 + """ + + model_config = ConfigDict(alias_generator=to_camel) + + total_count: int = Field(default=0, ge=0, description='文件总数') + total_size: int = Field(default=0, ge=0, description='文件总大小') + public_size: int = Field(default=0, ge=0, description='公开文件总大小') + private_size: int = Field(default=0, ge=0, description='受保护文件总大小') + active_count: int = Field(default=0, ge=0, description='有效文件数') + deleted_count: int = Field(default=0, ge=0, description='回收站文件数') + expired_count: int = Field(default=0, ge=0, description='已过期文件数') + retention_expiring_count: int = Field(default=0, ge=0, description='保留期限即将到期文件数') + acl_expiring_count: int = Field(default=0, ge=0, description='ACL即将过期文件数') + + +class FileAccessLogQueryModel(BaseModel): + """ + 文件访问审计不分页查询模型 + """ + + model_config = ConfigDict(alias_generator=to_camel) + + action: ( + Literal[ + 'upload', + 'download', + 'acl_update', + 'transfer', + 'delete', + 'restore', + 'purge', + 'reconcile', + 'retention_extend', + 'retention_dispose', + ] + | None + ) = Field(default=None, description='操作类型') + result: Literal['allowed', 'denied', 'completed', 'failed'] | None = Field(default=None, description='操作结果') + actor_name: str | None = Field(default=None, description='操作用户名称') + begin_time: str | None = Field(default=None, description='开始时间') + end_time: str | None = Field(default=None, description='结束时间') + + +class FileAccessLogPageQueryModel(FileAccessLogQueryModel): + """ + 文件访问审计分页查询模型 + """ + + page_num: int = Field(default=1, description='当前页码') + page_size: int = Field(default=10, description='每页记录数') + + +class DeleteFileModel(BaseModel): + """ + 删除文件模型 + """ + + model_config = ConfigDict(alias_generator=to_camel) + + file_ids: str = Field(description='需要删除的文件ID') + + +class TransferFileModel(BaseModel): + """ + 转移文件模型 + """ + + model_config = ConfigDict(alias_generator=to_camel, str_strip_whitespace=True) + + owner_user_id: int = Field(gt=0, description='新所有者用户ID') + dept_id: int = Field(gt=0, description='新所属部门ID') + retain_uploader_access: bool = Field(default=True, description='是否保留上传人访问权限') + reason: str = Field(min_length=1, max_length=500, description='转移原因') + + +class FileReconcileRunModel(BaseModel): + """ + 文件存储对账任务展示模型 + """ + + model_config = ConfigDict(alias_generator=to_camel, from_attributes=True) + + run_id: str = Field(description='任务ID') + trigger_type: Literal['manual', 'scheduled'] = Field(description='触发类型') + status: Literal['running', 'completed', 'failed'] = Field(description='任务状态') + check_hash: bool = Field(default=False, description='是否校验文件摘要') + scanned_file_count: int = Field(default=0, ge=0, description='扫描文件记录数') + scanned_storage_count: int = Field(default=0, ge=0, description='扫描物理文件数') + issue_count: int = Field(default=0, ge=0, description='发现异常数') + new_issue_count: int = Field(default=0, ge=0, description='新增或重新出现异常数') + resolved_issue_count: int = Field(default=0, ge=0, description='自动恢复异常数') + started_by: str | None = Field(default=None, description='发起人') + started_time: datetime = Field(description='开始时间') + finished_time: datetime | None = Field(default=None, description='完成时间') + error_message: str | None = Field(default=None, description='失败原因') + + +class FileReconcileRunPageQueryModel(BaseModel): + """ + 文件存储对账任务分页查询模型 + """ + + model_config = ConfigDict(alias_generator=to_camel) + + status: Literal['running', 'completed', 'failed'] | None = Field(default=None, description='任务状态') + trigger_type: Literal['manual', 'scheduled'] | None = Field(default=None, description='触发类型') + page_num: int = Field(default=1, ge=1, description='当前页码') + page_size: int = Field(default=10, ge=1, le=100, description='每页记录数') + + +class FileReconcileIssueModel(BaseModel): + """ + 文件存储对账异常展示模型 + """ + + model_config = ConfigDict(alias_generator=to_camel, from_attributes=True) + + issue_id: int = Field(description='异常ID') + issue_key: str = Field(description='异常唯一标识') + last_run_id: str = Field(description='最近发现任务ID') + issue_type: Literal[ + 'invalid_metadata', + 'missing_file', + 'unexpected_trash', + 'unexpected_source', + 'duplicate_file', + 'wrong_storage_root', + 'size_mismatch', + 'hash_mismatch', + 'orphan_file', + 'unsafe_entry', + ] = Field(description='异常类型') + severity: Literal['critical', 'warning', 'info'] = Field(description='严重级别') + file_id: str | None = Field(default=None, description='文件ID') + original_name: str | None = Field(default=None, description='原始文件名') + storage_type: str | None = Field(default=None, description='存储类型') + access_type: str | None = Field(default=None, description='访问类型') + expected_root: str | None = Field(default=None, description='预期存储区域') + expected_key: str | None = Field(default=None, description='预期相对路径') + actual_root: str | None = Field(default=None, description='实际存储区域') + actual_key: str | None = Field(default=None, description='实际相对路径') + expected_size: int | None = Field(default=None, ge=0, description='预期文件大小') + actual_size: int | None = Field(default=None, ge=0, description='实际文件大小') + expected_hash: str | None = Field(default=None, description='预期SHA-256') + actual_hash: str | None = Field(default=None, description='实际SHA-256') + status: Literal['open', 'ignored', 'quarantined', 'resolved'] = Field(description='处理状态') + detail: str | None = Field(default=None, description='异常说明') + occurrence_count: int = Field(default=1, ge=1, description='发现次数') + first_seen_time: datetime = Field(description='首次发现时间') + last_seen_time: datetime = Field(description='最近发现时间') + handle_action: str | None = Field(default=None, description='处理动作') + handle_reason: str | None = Field(default=None, description='处理原因') + handled_by: str | None = Field(default=None, description='处理人') + handled_time: datetime | None = Field(default=None, description='处理时间') + quarantine_key: str | None = Field(default=None, description='隔离区相对路径') + available_actions: list[FileReconcileAction] = Field(default_factory=list, description='可用处理动作') + + +class FileReconcileIssuePageQueryModel(BaseModel): + """ + 文件存储对账异常分页查询模型 + """ + + model_config = ConfigDict(alias_generator=to_camel) + + issue_type: str | None = Field(default=None, max_length=32, description='异常类型') + severity: Literal['critical', 'warning', 'info'] | None = Field(default=None, description='严重级别') + status: Literal['open', 'ignored', 'quarantined', 'resolved'] | None = Field(default=None, description='处理状态') + keyword: str | None = Field(default=None, max_length=100, description='文件或路径关键字') + page_num: int = Field(default=1, ge=1, description='当前页码') + page_size: int = Field(default=10, ge=1, le=100, description='每页记录数') + + +class FileReconcileStartModel(BaseModel): + """ + 启动文件存储对账模型 + """ + + model_config = ConfigDict(alias_generator=to_camel) + + check_hash: bool = Field(default=False, description='是否校验文件SHA-256') + + +class FileReconcileHandleModel(BaseModel): + """ + 文件存储对账异常处理模型 + """ + + model_config = ConfigDict(alias_generator=to_camel, str_strip_whitespace=True) + + action: FileReconcileAction = Field(description='处理动作') + reason: str = Field(min_length=1, max_length=500, description='处理原因') + original_name: str | None = Field(default=None, min_length=1, max_length=255, description='登记原始文件名') + + +class FileReconcileStatsModel(BaseModel): + """ + 文件存储对账统计模型 + """ + + model_config = ConfigDict(alias_generator=to_camel) + + open_count: int = Field(default=0, ge=0, description='待处理异常数') + critical_count: int = Field(default=0, ge=0, description='严重异常数') + warning_count: int = Field(default=0, ge=0, description='警告异常数') + ignored_count: int = Field(default=0, ge=0, description='已忽略异常数') + quarantined_count: int = Field(default=0, ge=0, description='隔离文件数') + latest_run: FileReconcileRunModel | None = Field(default=None, description='最近对账任务') diff --git a/ruoyi-fastapi-backend/module_admin/service/common_service.py b/ruoyi-fastapi-backend/module_admin/service/common_service.py index 254ad9f..c944aa0 100644 --- a/ruoyi-fastapi-backend/module_admin/service/common_service.py +++ b/ruoyi-fastapi-backend/module_admin/service/common_service.py @@ -1,14 +1,28 @@ -import os +import asyncio +import hashlib +import uuid +from collections.abc import AsyncGenerator from datetime import datetime +from pathlib import Path +from typing import Literal import aiofiles from fastapi import BackgroundTasks, Request, UploadFile +from sqlalchemy import ColumnElement +from sqlalchemy.ext.asyncio import AsyncSession from common.vo import CrudResponseModel from config.env import UploadConfig -from exceptions.exception import ServiceException +from exceptions.exception import FileRangeNotSatisfiableException, ServiceException +from module_admin.dao.file_access_dao import FileAclDao +from module_admin.dao.file_info_dao import FileInfoDao +from module_admin.entity.do.file_do import SysFileInfo from module_admin.entity.vo.common_vo import UploadResponseModel -from utils.upload_util import UploadUtil +from module_admin.entity.vo.file_vo import FileInfoModel +from module_admin.entity.vo.user_vo import CurrentUserModel +from module_admin.service.file_access_service import FileAuditService +from utils.file_util import FileByteRange, FileDownloadResult, FileUtil +from utils.upload_util import FilePathUtil, UploadUtil class CommonService: @@ -17,83 +31,533 @@ class CommonService: """ @classmethod - async def upload_service(cls, request: Request, file: UploadFile) -> CrudResponseModel: + async def upload_service( + cls, + request: Request, + query_db: AsyncSession, + current_user: CurrentUserModel, + file: UploadFile, + access_type: Literal['public', 'private'] = 'public', + ) -> CrudResponseModel: """ 通用上传service :param request: Request对象 + :param query_db: orm对象 + :param current_user: 当前用户对象 :param file: 上传文件对象 + :param access_type: 文件访问类型 :return: 上传结果 """ + if access_type not in {'public', 'private'}: + raise ServiceException(message='文件访问类型不合法') if not UploadUtil.check_file_extension(file): raise ServiceException(message='文件类型不合法') - relative_path = ( - f'upload/{datetime.now().strftime("%Y")}/{datetime.now().strftime("%m")}/{datetime.now().strftime("%d")}' - ) - dir_path = os.path.join(UploadConfig.UPLOAD_PATH, relative_path) + if file.size is not None and file.size > UploadConfig.MAX_FILE_SIZE: + raise ServiceException(message=f'文件大小不能超过{UploadConfig.MAX_FILE_SIZE // 1024 // 1024}MB') + + now = datetime.now() + relative_path = Path('upload', now.strftime('%Y'), now.strftime('%m'), now.strftime('%d')) + storage_root = UploadConfig.UPLOAD_PATH if access_type == 'public' else UploadConfig.PRIVATE_UPLOAD_PATH + dir_path = Path(storage_root, relative_path) + UploadUtil.ensure_directory(dir_path) + file_id = str(uuid.uuid4()) + extension = UploadUtil.get_file_extension(file.filename) + original_filename = UploadUtil.get_original_filename(file.filename) + file_stem = UploadUtil.get_safe_file_stem(file.filename) + for _ in range(10): + filename = f'{file_stem}_{now.strftime("%Y%m%d%H%M%S")}{UploadConfig.UPLOAD_MACHINE}{UploadUtil.generate_random_number()}.{extension}' + filepath = dir_path / filename + try: + total_size, file_hash = await cls._write_uploaded_file(file, filepath) + break + except FileExistsError: + continue + else: + raise ServiceException(message='文件名生成冲突，请重新上传') + + relative_path_url = relative_path.as_posix() + storage_key = f'{relative_path_url}/{filename}' + user = current_user.user + if user is None: + UploadUtil.delete_file(filepath) + raise ServiceException(message='无法获取当前用户信息') + try: - os.makedirs(dir_path) - except FileExistsError: - pass - filename = f'{file.filename.rsplit(".", 1)[0]}_{datetime.now().strftime("%Y%m%d%H%M%S")}{UploadConfig.UPLOAD_MACHINE}{UploadUtil.generate_random_number()}.{file.filename.rsplit(".")[-1]}' - filepath = os.path.join(dir_path, filename) - async with aiofiles.open(filepath, 'wb') as f: - # 流式写出大型文件，这里的10代表10MB - while True: - chunk = await file.read(1024 * 1024 * 10) - if not chunk: - break - await f.write(chunk) + file_info = FileInfoModel( + fileId=file_id, + originalName=original_filename, + storedName=filename, + storageKey=storage_key, + accessType=access_type, + uploadUserId=user.user_id, + ownerUserId=user.user_id, + deptId=user.dept_id, + extension=extension, + contentType=file.content_type, + fileSize=total_size, + fileHash=file_hash, + createBy=user.user_name, + createTime=now, + updateBy=user.user_name, + updateTime=now, + ) + await FileInfoDao.add_file_info_dao(query_db, file_info) + await query_db.commit() + except Exception: + await query_db.rollback() + if UploadUtil.check_file_exists(filepath): + UploadUtil.delete_file(filepath) + await cls._enqueue_file_access_log( + request, + current_user, + file_id, + action='upload', + result='failed', + error_message='文件信息写入失败', + ) + raise + + download_path = f'/common/files/{file_id}/download/{filename}' + if access_type == 'public': + file_name = f'{UploadConfig.UPLOAD_PREFIX}/{storage_key}' + file_url = f'{request.base_url}{UploadConfig.UPLOAD_PREFIX[1:]}/{storage_key}' + else: + file_name = download_path + file_url = f'{request.base_url}{download_path.lstrip("/")}' + + await cls._enqueue_file_access_log( + request, + current_user, + file_id, + action='upload', + result='completed', + bytes_sent=total_size, + ) return CrudResponseModel( is_success=True, result=UploadResponseModel( - fileName=f'{UploadConfig.UPLOAD_PREFIX}/{relative_path}/{filename}', + fileName=file_name, newFileName=filename, - originalFilename=file.filename, - url=f'{request.base_url}{UploadConfig.UPLOAD_PREFIX[1:]}/{relative_path}/{filename}', + originalFilename=original_filename, + url=file_url, + fileId=file_id, + accessType=access_type, + downloadUrl=download_path, ), message='上传成功', ) + @classmethod + async def _write_uploaded_file(cls, file: UploadFile, filepath: Path) -> tuple[int, str]: + """ + 将上传文件写入目标路径并计算摘要 + + :param file: 上传文件对象 + :param filepath: 文件目标路径 + :return: 文件大小和SHA-256 + """ + total_size = 0 + file_hasher = hashlib.sha256() + file_created = False + try: + async with aiofiles.open(filepath, 'xb') as target_file: + file_created = True + while chunk := await file.read(1024 * 1024): + total_size += len(chunk) + if total_size > UploadConfig.MAX_FILE_SIZE: + raise ServiceException( + message=f'文件大小不能超过{UploadConfig.MAX_FILE_SIZE // 1024 // 1024}MB' + ) + file_hasher.update(chunk) + await target_file.write(chunk) + except Exception: + if file_created and UploadUtil.check_file_exists(filepath): + UploadUtil.delete_file(filepath) + raise + return total_size, file_hasher.hexdigest() + + @classmethod + async def download_managed_file_services( + cls, + request: Request, + query_db: AsyncSession, + current_user: CurrentUserModel, + file_id: str, + enforce_owner_permission: bool = True, + file_data_scope_sql: ColumnElement | None = None, + range_header: str | None = None, + ) -> FileDownloadResult: + """ + 下载已登记文件service + + :param request: Request对象 + :param query_db: orm对象 + :param current_user: 当前用户对象 + :param file_id: 文件ID + :param enforce_owner_permission: 是否校验文件所有者权限 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :param range_header: Range请求头 + :return: 文件下载结果 + """ + file_info = await FileInfoDao.get_file_info_by_id(query_db, file_id, file_data_scope_sql) + user = current_user.user + if file_info is None or user is None: + await cls._enqueue_file_access_log( + request, + current_user, + file_id, + action='download', + result='denied', + error_message='文件不存在或无权访问', + ) + raise ServiceException(message='文件不存在或无权访问') + if file_info.storage_type != 'local' or file_info.access_type not in {'public', 'private'}: + await cls._enqueue_file_access_log( + request, + current_user, + file_id, + action='download', + result='failed', + error_message='文件存储类型或访问类型异常', + ) + raise ServiceException(message='文件不存在或无权访问') + + current_time = datetime.now() + is_expired = ( + file_info.access_type == 'private' and file_info.expire_time and file_info.expire_time < current_time + ) + if is_expired: + is_allowed = False + elif not enforce_owner_permission or file_info.access_type == 'public': + is_allowed = True + else: + is_allowed = await cls._has_private_file_download_permission( + query_db, + current_user, + file_info, + file_id, + current_time, + ) + if not is_allowed: + await cls._enqueue_file_access_log( + request, + current_user, + file_id, + action='download', + result='denied', + error_message='文件不存在或无权访问', + ) + raise ServiceException(message='文件不存在或无权访问') + + storage_root = ( + UploadConfig.UPLOAD_PATH if file_info.access_type == 'public' else UploadConfig.PRIVATE_UPLOAD_PATH + ) + try: + filepath = FilePathUtil.resolve_file_within_root(storage_root, file_info.storage_key) + except (FileNotFoundError, ValueError) as exc: + await cls._enqueue_file_access_log( + request, + current_user, + file_id, + action='download', + result='failed', + error_message='文件不存在或存储路径异常', + ) + raise ServiceException(message='文件不存在或无权访问') from exc + try: + byte_range = FileUtil.parse_byte_range(range_header, filepath.stat().st_size) + except FileRangeNotSatisfiableException: + await cls._enqueue_file_access_log( + request, + current_user, + file_id, + action='download', + result='failed', + error_message='RangeNotSatisfiable', + operation_detail={'range': range_header or ''}, + ) + raise + + original_name = file_info.original_name + await query_db.rollback() + await cls._enqueue_file_access_log( + request, + current_user, + file_id, + action='download', + result='allowed', + operation_detail=cls._build_download_operation_detail(byte_range), + ) + stream = cls._generate_audited_file(request, current_user, file_id, filepath, byte_range) + return FileDownloadResult( + data=stream, + filename=original_name, + byte_range=byte_range, + ) + + @classmethod + async def _has_private_file_download_permission( + cls, + query_db: AsyncSession, + current_user: CurrentUserModel, + file_info: SysFileInfo, + file_id: str, + current_time: datetime, + ) -> bool: + """ + 校验私有文件下载权限 + + :param query_db: orm对象 + :param current_user: 当前用户对象 + :param file_info: 文件信息 + :param file_id: 文件ID + :param current_time: 当前时间 + :return: 是否允许下载 + """ + user = current_user.user + if user is None or user.user_id is None: + return False + if bool(getattr(user, 'admin', False)) or user.user_id == file_info.owner_user_id: + return True + + file_acl_list = await FileAclDao.get_effective_file_acl_list(query_db, file_id, current_time) + role_ids = cls._get_current_user_role_ids(user) + dept_id, ancestor_dept_ids = cls._get_current_user_dept_ids(user) + matched_effects = [] + for file_acl in file_acl_list: + is_matched = False + if file_acl.subject_type == 'user': + is_matched = file_acl.subject_id == user.user_id + elif file_acl.subject_type == 'role': + is_matched = file_acl.subject_id in role_ids + elif file_acl.subject_type == 'dept': + is_matched = file_acl.subject_id == dept_id or ( + file_acl.include_children in {'1', True} and file_acl.subject_id in ancestor_dept_ids + ) + if is_matched: + matched_effects.append(file_acl.effect) + + if 'deny' in matched_effects: + return False + if user.user_id == file_info.upload_user_id and getattr(file_info, 'uploader_access_enabled', '1') in { + '1', + True, + }: + return True + return 'allow' in matched_effects + + @staticmethod + def _get_current_user_role_ids(user: object) -> set[int]: + """ + 获取当前用户角色ID集合 + + :param user: 当前用户对象 + :return: 角色ID集合 + """ + role_ids = { + role.role_id + for role in (getattr(user, 'role', None) or []) + if role is not None and getattr(role, 'role_id', None) is not None + } + role_ids_text = getattr(user, 'role_ids', None) + if role_ids_text: + role_ids.update(int(role_id) for role_id in role_ids_text.split(',') if role_id.strip().isdigit()) + return role_ids + + @staticmethod + def _get_current_user_dept_ids(user: object) -> tuple[int | None, set[int]]: + """ + 获取当前用户部门及祖级部门ID + + :param user: 当前用户对象 + :return: 当前部门ID和祖级部门ID集合 + """ + dept = getattr(user, 'dept', None) + dept_id = getattr(user, 'dept_id', None) or getattr(dept, 'dept_id', None) + ancestors = getattr(dept, 'ancestors', None) or '' + ancestor_dept_ids = {int(ancestor_id) for ancestor_id in ancestors.split(',') if ancestor_id.strip().isdigit()} + return dept_id, ancestor_dept_ids + + @classmethod + async def _generate_audited_file( + cls, + request: Request, + current_user: CurrentUserModel, + file_id: str, + filepath: Path, + byte_range: FileByteRange, + ) -> AsyncGenerator[bytes, None]: + """ + 生成带有完成审计的文件流 + + :param request: Request对象 + :param current_user: 当前用户对象 + :param file_id: 文件ID + :param filepath: 文件路径 + :param byte_range: 文件字节范围 + :yield: 文件二进制数据 + """ + bytes_sent = 0 + audit_result: Literal['completed', 'failed'] = 'failed' + error_message = 'StreamClosed' + try: + async for chunk in UploadUtil.generate_file( + filepath, + start=byte_range.start, + length=byte_range.length, + ): + bytes_sent += len(chunk) + yield chunk + if bytes_sent != byte_range.length: + raise OSError('文件在下载期间发生变化') + except asyncio.CancelledError: + error_message = 'CancelledError' + raise + except Exception as exc: + error_message = exc.__class__.__name__ + raise + else: + audit_result = 'completed' + error_message = '' + finally: + await cls._enqueue_file_access_log( + request, + current_user, + file_id, + action='download', + result=audit_result, + bytes_sent=bytes_sent, + error_message=error_message, + operation_detail=cls._build_download_operation_detail(byte_range), + ) + + @classmethod + async def _enqueue_file_access_log( + cls, + request: Request, + current_user: CurrentUserModel, + file_id: str, + action: Literal['upload', 'download'], + result: Literal['allowed', 'denied', 'completed', 'failed'], + bytes_sent: int = 0, + error_message: str = '', + operation_detail: dict[str, object] | None = None, + ) -> None: + """ + 将文件访问审计写入日志队列 + + :param request: Request对象 + :param current_user: 当前用户对象 + :param file_id: 文件ID + :param action: 操作类型 + :param result: 操作结果 + :param bytes_sent: 已发送字节数 + :param error_message: 失败原因 + :param operation_detail: 操作详情 + :return: None + """ + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_id, + action, + result, + bytes_sent=bytes_sent, + error_message=error_message, + operation_detail=operation_detail, + ) + + @staticmethod + def _build_download_operation_detail(byte_range: FileByteRange) -> dict[str, object] | None: + """ + 构造分段下载审计详情 + + :param byte_range: 文件字节范围 + :return: 分段下载审计详情 + """ + if not byte_range.is_partial: + return None + return { + 'rangeStart': byte_range.start, + 'rangeEnd': byte_range.end, + 'fileSize': byte_range.file_size, + } + @classmethod async def download_services( - cls, background_tasks: BackgroundTasks, file_name: str, delete: bool - ) -> CrudResponseModel: + cls, + background_tasks: BackgroundTasks, + file_name: str, + delete: bool, + range_header: str | None = None, + ) -> FileDownloadResult: """ 下载下载目录文件service :param background_tasks: 后台任务对象 :param file_name: 下载的文件名称 :param delete: 是否在下载完成后删除文件 - :return: 上传结果 + :param range_header: Range请求头 + :return: 文件下载结果 """ - filepath = os.path.join(UploadConfig.DOWNLOAD_PATH, file_name) - if '..' in file_name: - raise ServiceException(message='文件名称不合法') - if not UploadUtil.check_file_exists(filepath): - raise ServiceException(message='文件不存在') + try: + filepath = FilePathUtil.resolve_file_within_root(UploadConfig.DOWNLOAD_PATH, file_name) + except (FileNotFoundError, ValueError) as exc: + raise ServiceException(message='文件名称不合法或文件不存在') from exc + accept_ranges = not delete + byte_range = FileUtil.parse_byte_range(range_header if accept_ranges else None, filepath.stat().st_size) if delete: background_tasks.add_task(UploadUtil.delete_file, filepath) - return CrudResponseModel(is_success=True, result=UploadUtil.generate_file(filepath), message='下载成功') + return FileDownloadResult( + data=UploadUtil.generate_file( + filepath, + start=byte_range.start, + length=byte_range.length, + ), + filename=file_name, + byte_range=byte_range, + accept_ranges=accept_ranges, + ) @classmethod - async def download_resource_services(cls, resource: str) -> CrudResponseModel: + async def download_resource_services( + cls, + resource: str, + range_header: str | None = None, + ) -> FileDownloadResult: """ 下载上传目录文件service :param resource: 下载的文件名称 - :return: 上传结果 + :param range_header: Range请求头 + :return: 文件下载结果 """ - filepath = os.path.join(resource.replace(UploadConfig.UPLOAD_PREFIX, UploadConfig.UPLOAD_PATH)) - filename = resource.rsplit('/', 1)[-1] + resource_prefix = f'{UploadConfig.UPLOAD_PREFIX.rstrip("/")}/' + if not resource.startswith(resource_prefix): + raise ServiceException(message='资源路径不合法') + relative_resource = resource[len(resource_prefix) :] + try: + filepath = FilePathUtil.resolve_file_within_root(UploadConfig.UPLOAD_PATH, relative_resource) + except (FileNotFoundError, ValueError) as exc: + raise ServiceException(message='资源路径不合法或文件不存在') from exc + filename = filepath.name if ( '..' in filename or not UploadUtil.check_file_timestamp(filename) or not UploadUtil.check_file_machine(filename) or not UploadUtil.check_file_random_code(filename) + or UploadUtil.get_file_extension(filename) not in UploadConfig.DEFAULT_ALLOWED_EXTENSION ): - raise ServiceException(message='文件名称不合法') - if not UploadUtil.check_file_exists(filepath): - raise ServiceException(message='文件不存在') - return CrudResponseModel(is_success=True, result=UploadUtil.generate_file(filepath), message='下载成功') + raise ServiceException(message='资源文件名称不合法') + byte_range = FileUtil.parse_byte_range(range_header, filepath.stat().st_size) + return FileDownloadResult( + data=UploadUtil.generate_file( + filepath, + start=byte_range.start, + length=byte_range.length, + ), + filename=filename, + byte_range=byte_range, + ) diff --git a/ruoyi-fastapi-backend/module_admin/service/file_access_service.py b/ruoyi-fastapi-backend/module_admin/service/file_access_service.py new file mode 100644 index 0000000..2bec7c5 --- /dev/null +++ b/ruoyi-fastapi-backend/module_admin/service/file_access_service.py @@ -0,0 +1,572 @@ +import json +from datetime import datetime +from typing import Any, Literal + +from fastapi import Request +from sqlalchemy import ColumnElement +from sqlalchemy.ext.asyncio import AsyncSession + +from common.vo import CrudResponseModel +from exceptions.exception import ServiceException +from middlewares.trace_middleware.ctx import TraceCtx +from module_admin.dao.file_access_dao import FileAccessLogDao, FileAclDao +from module_admin.dao.file_info_dao import FileInfoDao +from module_admin.entity.do.file_do import SysFileAcl +from module_admin.entity.vo.dept_vo import DeptTreeModel +from module_admin.entity.vo.file_vo import ( + BatchSaveFileAclModel, + FileAccessLogModel, + FileAclBuiltinPermissionModel, + FileAclListModel, + FileAclModel, + FileAclSubjectOptionModel, + SaveFileAclModel, +) +from module_admin.entity.vo.user_vo import CurrentUserModel +from module_admin.service.dept_service import DeptService +from module_admin.service.log_service import LogQueueService +from utils.client_ip_util import ClientIPUtil +from utils.file_util import FileUtil +from utils.log_util import LogSanitizer, logger + + +class FileAclService: + @classmethod + async def get_file_acl_list_services( + cls, + query_db: AsyncSession, + file_id: str, + file_data_scope_sql: ColumnElement, + user_data_scope_sql: ColumnElement, + dept_data_scope_sql: ColumnElement, + ) -> FileAclListModel: + """ + 获取文件访问控制列表service + + :param query_db: orm对象 + :param file_id: 文件ID + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :param user_data_scope_sql: 用户数据权限对应的查询sql语句 + :param dept_data_scope_sql: 部门数据权限对应的查询sql语句 + :return: 文件访问控制列表及版本 + """ + file_info = await FileInfoDao.get_file_info_detail_by_id(query_db, file_id, file_data_scope_sql) + if file_info is None: + raise ServiceException(message='文件信息不存在或超出数据权限') + file_acl_list = await FileAclDao.get_file_acl_list(query_db, file_id) + subject_ids = cls._group_acl_subject_ids(file_acl_list) + if file_info.access_type == 'private' and file_info.owner_user_id: + subject_ids['user'].add(file_info.owner_user_id) + if file_info.access_type == 'private' and file_info.upload_user_id: + subject_ids['user'].add(file_info.upload_user_id) + subject_name_map = await FileAclDao.get_acl_subject_name_map( + query_db, + subject_ids, + user_data_scope_sql, + dept_data_scope_sql, + ) + return FileAclListModel( + aclVersion=file_info.acl_version, + builtinPermissions=cls._build_builtin_permissions(file_info, subject_name_map), + entries=[ + FileAclModel( + aclId=file_acl.acl_id, + fileId=file_acl.file_id, + subjectType=file_acl.subject_type, + subjectId=file_acl.subject_id, + subjectName=subject_name_map.get( + (file_acl.subject_type, file_acl.subject_id), + f'不可用或无权查看主体（{file_acl.subject_id}）', + ), + permission=file_acl.permission, + effect=file_acl.effect, + includeChildren=file_acl.include_children == '1', + expireTime=file_acl.expire_time, + createBy=file_acl.create_by, + createTime=file_acl.create_time, + ) + for file_acl in file_acl_list + ], + ) + + @staticmethod + def _build_builtin_permissions( + file_info: object, + subject_name_map: dict[tuple[str, int], str], + ) -> list[FileAclBuiltinPermissionModel]: + """ + 构建文件内置访问权限 + + :param file_info: 文件信息 + :param subject_name_map: 主体名称映射 + :return: 文件内置访问权限列表 + """ + if getattr(file_info, 'access_type', None) != 'private': + return [] + builtin_permissions = [ + FileAclBuiltinPermissionModel( + source='admin', + subjectName='平台管理员', + enabled=True, + denyOverridable=False, + description='平台管理员始终允许下载，显式拒绝不能覆盖。', + ) + ] + owner_user_id = getattr(file_info, 'owner_user_id', None) + upload_user_id = getattr(file_info, 'upload_user_id', None) + if owner_user_id: + builtin_permissions.append( + FileAclBuiltinPermissionModel( + source='owner', + subjectId=owner_user_id, + subjectName=subject_name_map.get( + ('user', owner_user_id), + f'不可用或无权查看主体（{owner_user_id}）', + ), + enabled=True, + denyOverridable=False, + description='文件所有者始终允许下载，显式拒绝不能覆盖。', + ) + ) + if upload_user_id: + uploader_is_owner = upload_user_id == owner_user_id + uploader_access_enabled = getattr(file_info, 'uploader_access_enabled', '1') in {'1', True} + builtin_permissions.append( + FileAclBuiltinPermissionModel( + source='uploader', + subjectId=upload_user_id, + subjectName=subject_name_map.get( + ('user', upload_user_id), + f'不可用或无权查看主体（{upload_user_id}）', + ), + enabled=uploader_access_enabled, + denyOverridable=uploader_access_enabled and not uploader_is_owner, + description=( + ( + '当前上传人同时是文件所有者，显式拒绝不能覆盖；所有权转移后按上传人规则判断。' + if uploader_is_owner + else '上传人默认允许下载，匹配的显式拒绝可以覆盖。' + ) + if uploader_access_enabled + else ( + '上传人访问权限已移除；当前用户仍通过文件所有者权限访问。' + if uploader_is_owner + else '上传人访问权限已在文件转移时移除。' + ) + ), + ) + ) + return builtin_permissions + + @classmethod + async def search_file_acl_subjects_services( + cls, + query_db: AsyncSession, + subject_type: str, + keyword: str | None, + limit: int, + user_data_scope_sql: ColumnElement, + dept_data_scope_sql: ColumnElement, + ) -> list[FileAclSubjectOptionModel]: + """ + 查询文件访问控制主体选项service + + :param query_db: orm对象 + :param subject_type: 主体类型 + :param keyword: 查询关键字 + :param limit: 返回数量限制 + :param user_data_scope_sql: 用户数据权限对应的查询sql语句 + :param dept_data_scope_sql: 部门数据权限对应的查询sql语句 + :return: 主体选项列表 + """ + subject_list = await FileAclDao.search_acl_subjects( + query_db, + subject_type, + keyword, + limit, + user_data_scope_sql, + dept_data_scope_sql, + ) + return [FileAclSubjectOptionModel.model_validate(item, by_name=True) for item in subject_list] + + @classmethod + async def get_file_acl_dept_tree_services( + cls, + query_db: AsyncSession, + dept_data_scope_sql: ColumnElement, + ) -> list[DeptTreeModel]: + """ + 获取文件授权部门树service + + :param query_db: orm对象 + :param dept_data_scope_sql: 部门数据权限对应的查询sql语句 + :return: 部门树 + """ + dept_list = await FileAclDao.get_acl_dept_list(query_db, dept_data_scope_sql) + return DeptService.list_to_tree(dept_list) + + @classmethod + async def save_file_acl_services( + cls, + query_db: AsyncSession, + current_user: CurrentUserModel, + file_id: str, + save_file_acl: SaveFileAclModel, + file_data_scope_sql: ColumnElement, + user_data_scope_sql: ColumnElement, + dept_data_scope_sql: ColumnElement, + request: Request | None = None, + ) -> CrudResponseModel: + """ + 保存文件访问控制service + + :param query_db: orm对象 + :param current_user: 当前用户对象 + :param file_id: 文件ID + :param save_file_acl: 文件访问控制参数 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :param user_data_scope_sql: 用户数据权限对应的查询sql语句 + :param dept_data_scope_sql: 部门数据权限对应的查询sql语句 + :param request: Request对象 + :return: 保存结果 + """ + file_info = await FileInfoDao.get_file_info_by_id_for_update(query_db, file_id, file_data_scope_sql) + if file_info is None: + raise ServiceException(message='文件信息不存在、已删除或超出数据权限') + if file_info.access_type != 'private': + raise ServiceException(message='公开文件不支持配置访问权限') + user = current_user.user + if user is None or not user.user_name: + raise ServiceException(message='无法获取当前用户信息') + current_acl_version = file_info.acl_version or 0 + if save_file_acl.acl_version != current_acl_version: + await query_db.rollback() + raise ServiceException(message='文件权限已被其他用户修改，请刷新后重试') + + current_time = datetime.now() + unique_subjects = set() + subject_ids: dict[str, set[int]] = {'user': set(), 'role': set(), 'dept': set()} + normalized_expire_times = [] + for entry in save_file_acl.entries: + subject_key = (entry.subject_type, entry.subject_id) + if subject_key in unique_subjects: + raise ServiceException(message='同一授权主体不能重复配置') + unique_subjects.add(subject_key) + subject_ids[entry.subject_type].add(entry.subject_id) + expire_time = entry.expire_time + if expire_time and expire_time.tzinfo: + expire_time = expire_time.astimezone().replace(tzinfo=None) + normalized_expire_times.append(expire_time) + if expire_time and expire_time <= current_time: + raise ServiceException(message='授权过期时间必须晚于当前时间') + + subject_name_map = await FileAclDao.get_acl_subject_name_map( + query_db, + subject_ids, + user_data_scope_sql, + dept_data_scope_sql, + ) + if len(subject_name_map) != len(unique_subjects): + raise ServiceException(message='部分授权主体不存在、已停用或超出数据权限') + + file_acl_list = [ + SysFileAcl( + file_id=file_id, + subject_type=entry.subject_type, + subject_id=entry.subject_id, + permission='download', + effect=entry.effect, + include_children='1' if entry.subject_type == 'dept' and entry.include_children else '0', + expire_time=expire_time, + create_by=user.user_name, + create_time=current_time, + del_flag='0', + ) + for entry, expire_time in zip(save_file_acl.entries, normalized_expire_times, strict=True) + ] + try: + await FileAclDao.replace_file_acl_list(query_db, file_id, file_acl_list) + file_info.acl_version = current_acl_version + 1 + file_info.update_by = user.user_name + file_info.update_time = current_time + await query_db.commit() + except Exception as exc: + await query_db.rollback() + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_id, + 'acl_update', + 'failed', + error_message=exc.__class__.__name__, + operation_detail={'previousAclVersion': current_acl_version}, + ) + raise + subject_type_counts = { + subject_type: len(subject_ids[subject_type]) for subject_type in ('user', 'role', 'dept') + } + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_id, + 'acl_update', + 'completed', + operation_detail={ + 'previousAclVersion': current_acl_version, + 'newAclVersion': current_acl_version + 1, + 'entryCount': len(save_file_acl.entries), + 'allowCount': sum(entry.effect == 'allow' for entry in save_file_acl.entries), + 'denyCount': sum(entry.effect == 'deny' for entry in save_file_acl.entries), + 'subjectTypeCounts': subject_type_counts, + }, + ) + return CrudResponseModel(is_success=True, message='文件权限保存成功') + + @classmethod + async def batch_save_file_acl_services( + cls, + query_db: AsyncSession, + current_user: CurrentUserModel, + batch_save_file_acl: BatchSaveFileAclModel, + file_data_scope_sql: ColumnElement, + user_data_scope_sql: ColumnElement, + dept_data_scope_sql: ColumnElement, + request: Request | None = None, + ) -> CrudResponseModel: + """ + 批量保存文件访问控制service + + :param query_db: orm对象 + :param current_user: 当前用户对象 + :param batch_save_file_acl: 批量文件访问控制参数 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :param user_data_scope_sql: 用户数据权限对应的查询sql语句 + :param dept_data_scope_sql: 部门数据权限对应的查询sql语句 + :param request: Request对象 + :return: 保存结果 + """ + file_ids = FileUtil.parse_file_ids(batch_save_file_acl.file_ids) + user = current_user.user + if user is None or not user.user_name: + raise ServiceException(message='无法获取当前用户信息') + file_infos = await FileInfoDao.get_file_infos_by_ids_for_update(query_db, file_ids, file_data_scope_sql) + if len(file_infos) != len(file_ids): + await query_db.rollback() + raise ServiceException(message='部分文件不存在、已删除或超出数据权限') + if any(file_info.access_type != 'private' for file_info in file_infos): + await query_db.rollback() + raise ServiceException(message='批量授权仅支持受保护文件') + + current_time = datetime.now() + unique_subjects = set() + subject_ids: dict[str, set[int]] = {'user': set(), 'role': set(), 'dept': set()} + normalized_entries = [] + for entry in batch_save_file_acl.entries: + subject_key = (entry.subject_type, entry.subject_id) + if subject_key in unique_subjects: + raise ServiceException(message='同一授权主体不能重复配置') + unique_subjects.add(subject_key) + subject_ids[entry.subject_type].add(entry.subject_id) + expire_time = entry.expire_time + if expire_time and expire_time.tzinfo: + expire_time = expire_time.astimezone().replace(tzinfo=None) + if expire_time and expire_time <= current_time: + raise ServiceException(message='授权过期时间必须晚于当前时间') + normalized_entries.append((entry, expire_time)) + + subject_name_map = await FileAclDao.get_acl_subject_name_map( + query_db, + subject_ids, + user_data_scope_sql, + dept_data_scope_sql, + ) + if len(subject_name_map) != len(unique_subjects): + raise ServiceException(message='部分授权主体不存在、已停用或超出数据权限') + + acl_versions = {file_info.file_id: file_info.acl_version or 0 for file_info in file_infos} + file_acl_list = [ + SysFileAcl( + file_id=file_id, + subject_type=entry.subject_type, + subject_id=entry.subject_id, + permission='download', + effect=entry.effect, + include_children='1' if entry.subject_type == 'dept' and entry.include_children else '0', + expire_time=expire_time, + create_by=user.user_name, + create_time=current_time, + del_flag='0', + ) + for file_id in file_ids + for entry, expire_time in normalized_entries + ] + try: + await FileAclDao.replace_file_acl_lists(query_db, file_ids, file_acl_list) + for file_info in file_infos: + file_info.acl_version = acl_versions[file_info.file_id] + 1 + file_info.update_by = user.user_name + file_info.update_time = current_time + await query_db.commit() + except Exception as exc: + await query_db.rollback() + for file_id in file_ids: + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_id, + 'acl_update', + 'failed', + error_message=exc.__class__.__name__, + operation_detail={'batch': True, 'previousAclVersion': acl_versions[file_id]}, + ) + raise + + subject_type_counts = { + subject_type: len(subject_ids[subject_type]) for subject_type in ('user', 'role', 'dept') + } + for file_id in file_ids: + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_id, + 'acl_update', + 'completed', + operation_detail={ + 'batch': True, + 'previousAclVersion': acl_versions[file_id], + 'newAclVersion': acl_versions[file_id] + 1, + 'entryCount': len(batch_save_file_acl.entries), + 'allowCount': sum(entry.effect == 'allow' for entry in batch_save_file_acl.entries), + 'denyCount': sum(entry.effect == 'deny' for entry in batch_save_file_acl.entries), + 'subjectTypeCounts': subject_type_counts, + }, + ) + return CrudResponseModel(is_success=True, message='文件权限批量保存成功') + + @staticmethod + def _group_acl_subject_ids(file_acl_list: list[SysFileAcl]) -> dict[str, set[int]]: + """ + 按类型分组文件访问控制主体ID + + :param file_acl_list: 文件访问控制列表 + :return: 主体ID分组 + """ + subject_ids: dict[str, set[int]] = {'user': set(), 'role': set(), 'dept': set()} + for file_acl in file_acl_list: + if file_acl.subject_type in subject_ids: + subject_ids[file_acl.subject_type].add(file_acl.subject_id) + return subject_ids + + +class FileAuditService: + """ + 文件审计服务层 + """ + + @classmethod + async def enqueue_file_audit( + cls, + request: Request | None, + current_user: CurrentUserModel, + file_id: str, + action: Literal[ + 'upload', + 'download', + 'acl_update', + 'transfer', + 'delete', + 'restore', + 'purge', + 'reconcile', + 'retention_extend', + 'retention_dispose', + ], + result: Literal['allowed', 'denied', 'completed', 'failed'], + bytes_sent: int = 0, + error_message: str = '', + operation_detail: dict[str, Any] | None = None, + ) -> None: + """ + 将文件审计事件写入日志队列 + + :param request: Request对象 + :param current_user: 当前用户对象 + :param file_id: 文件ID + :param action: 操作类型 + :param result: 操作结果 + :param bytes_sent: 已发送字节数 + :param error_message: 失败原因 + :param operation_detail: 操作详情 + :return: None + """ + if request is None: + return + try: + user = current_user.user + file_access_log = FileAccessLogModel( + fileId=file_id, + action=action, + actorUserId=user.user_id if user else None, + actorName=user.user_name if user else '', + result=result, + requestId=TraceCtx.get_request_id(), + traceId=TraceCtx.get_trace_id(), + ipAddress=ClientIPUtil.get_client_ip(request), + userAgent=(request.headers.get('User-Agent') or '')[:500], + bytesSent=bytes_sent, + errorMessage=error_message[:500], + operationDetail=cls._serialize_operation_detail(operation_detail), + accessTime=datetime.now(), + ) + await LogQueueService.enqueue_file_access_log( + request, + file_access_log, + source=f'file:{file_id}:{action}:{result}', + ) + except Exception as exc: + logger.error(f'文件审计写入队列失败: {exc}') + + @classmethod + async def add_system_file_audit( + cls, + query_db: AsyncSession, + file_id: str, + action: Literal['purge'], + result: Literal['completed', 'failed'], + error_message: str = '', + operation_detail: dict[str, Any] | None = None, + ) -> None: + """ + 在后台任务事务内写入文件审计记录 + + :param query_db: orm对象 + :param file_id: 文件ID + :param action: 操作类型 + :param result: 操作结果 + :param error_message: 失败原因 + :param operation_detail: 操作详情 + :return: None + """ + file_access_log = FileAccessLogModel( + fileId=file_id, + action=action, + actorName='system', + result=result, + errorMessage=error_message[:500], + operationDetail=cls._serialize_operation_detail(operation_detail), + accessTime=datetime.now(), + ) + await FileAccessLogDao.add_file_access_log_dao(query_db, file_access_log) + + @staticmethod + def _serialize_operation_detail(operation_detail: dict[str, Any] | None) -> str: + """ + 序列化文件操作详情 + + :param operation_detail: 操作详情 + :return: 序列化后的操作详情 + """ + if not operation_detail: + return '' + sanitized_detail = LogSanitizer.sanitize_data(operation_detail) + return json.dumps(sanitized_detail, ensure_ascii=False, default=str, separators=(',', ':')) diff --git a/ruoyi-fastapi-backend/module_admin/service/file_business_service.py b/ruoyi-fastapi-backend/module_admin/service/file_business_service.py new file mode 100644 index 0000000..6c7b0c7 --- /dev/null +++ b/ruoyi-fastapi-backend/module_admin/service/file_business_service.py @@ -0,0 +1,536 @@ +import uuid +from datetime import datetime, timedelta +from typing import Any + +from sqlalchemy import ColumnElement, true +from sqlalchemy.ext.asyncio import AsyncSession + +from common.vo import CrudResponseModel, PageModel +from exceptions.exception import ServiceException +from module_admin.dao.file_business_dao import ( + FileReferenceDao, + FileRetentionNoticeDao, + FileRetentionPolicyDao, +) +from module_admin.dao.file_info_dao import FileInfoDao +from module_admin.entity.do.file_do import ( + SysFileInfo, + SysFileReference, + SysFileRetentionNotice, + SysFileRetentionPolicy, +) +from module_admin.entity.vo.file_vo import ( + FileReferenceModel, + FileRetentionNoticePageQueryModel, + FileRetentionPolicyModel, + FileRetentionScanModel, +) +from utils.common_util import CamelCaseUtil + + +class FileReferenceService: + """ + 文件业务引用服务层 + """ + + MAX_REFERENCE_FILES = 100 + + @classmethod + async def get_file_reference_list_services( + cls, + query_db: AsyncSession, + file_id: str, + file_data_scope_sql: ColumnElement, + ) -> list[FileReferenceModel]: + """ + 获取文件业务引用列表service + + :param query_db: orm对象 + :param file_id: 文件ID + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :return: 文件业务引用列表 + """ + file_info = await FileInfoDao.get_file_management_detail_by_id(query_db, file_id, file_data_scope_sql) + if file_info is None: + raise ServiceException(message='文件信息不存在或超出数据权限') + file_reference_list = await FileReferenceDao.get_file_reference_list(query_db, file_id) + result = [FileReferenceModel(**CamelCaseUtil.transform_result(item)) for item in file_reference_list] + business_type = file_info.get('business_type') + business_id = file_info.get('business_id') + reference_keys = {(item.business_type, item.business_id) for item in file_reference_list} + if business_type and business_id and (business_type, business_id) not in reference_keys: + result.insert( + 0, + FileReferenceModel( + fileId=file_id, + businessType=business_type, + businessId=business_id, + legacy=True, + ), + ) + return result + + @classmethod + async def replace_business_file_references_services( + cls, + query_db: AsyncSession, + business_type: str, + business_id: str, + file_ids: list[str], + create_by: str, + file_data_scope_sql: ColumnElement, + business_name: str | None = None, + ) -> None: + """ + 在调用方业务事务内全量替换文件引用service + + :param query_db: orm对象 + :param business_type: 业务类型 + :param business_id: 业务ID + :param file_ids: 文件ID列表 + :param create_by: 创建者 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :param business_name: 业务名称 + :return: None + """ + normalized_business_type = cls._normalize_text(business_type, '业务类型', 50, required=True) + normalized_business_id = cls._normalize_text(business_id, '业务ID', 64, required=True) + normalized_business_name = cls._normalize_text(business_name, '业务名称', 255) + normalized_create_by = cls._normalize_text(create_by, '创建者', 64) or '' + normalized_file_ids = cls._normalize_file_ids(file_ids) + file_infos = [] + if normalized_file_ids: + file_infos = await FileInfoDao.get_file_infos_by_ids_for_update( + query_db, + normalized_file_ids, + file_data_scope_sql, + ) + if len(file_infos) != len(normalized_file_ids): + raise ServiceException(message='部分引用文件不存在或已失效') + create_time = datetime.now() + retention_policy = await FileRetentionPolicyService.get_enabled_file_retention_policy_services( + query_db, + normalized_business_type, + ) + if retention_policy and any( + getattr(file_info, 'access_type', 'private') != 'private' for file_info in file_infos + ): + raise ServiceException(message='配置保留策略的业务只能引用受保护文件') + retention_expire_time = ( + create_time + timedelta(days=retention_policy.retention_days) if retention_policy else None + ) + file_reference_list = [ + SysFileReference( + file_id=file_id, + business_type=normalized_business_type, + business_id=normalized_business_id, + business_name=normalized_business_name, + retention_expire_time=retention_expire_time, + create_by=normalized_create_by, + create_time=create_time, + ) + for file_id in normalized_file_ids + ] + await FileReferenceDao.replace_business_file_references( + query_db, + normalized_business_type, + normalized_business_id, + file_reference_list, + ) + + @classmethod + async def remove_business_file_references_services( + cls, + query_db: AsyncSession, + business_type: str, + business_id: str, + ) -> None: + """ + 在调用方业务事务内解除业务对象的全部文件引用service + + :param query_db: orm对象 + :param business_type: 业务类型 + :param business_id: 业务ID + :return: None + """ + await cls.replace_business_file_references_services( + query_db, + business_type, + business_id, + [], + create_by='', + file_data_scope_sql=true(), + ) + + @classmethod + async def get_file_reference_count_map_services( + cls, + query_db: AsyncSession, + file_infos: list[SysFileInfo] | list[dict], + ) -> dict[str, int]: + """ + 获取包含兼容字段的文件业务引用数量映射service + + :param query_db: orm对象 + :param file_infos: 文件信息列表 + :return: 文件ID和业务引用数量映射 + """ + file_ids = [str(cls._get_value(item, 'file_id', 'fileId')) for item in file_infos] + reference_count_map = await FileReferenceDao.get_file_reference_count_map(query_db, file_ids) + for file_info in file_infos: + file_id = str(cls._get_value(file_info, 'file_id', 'fileId')) + business_type = cls._get_value(file_info, 'business_type', 'businessType') + business_id = cls._get_value(file_info, 'business_id', 'businessId') + if business_type and business_id: + reference_count_map[file_id] = reference_count_map.get(file_id, 0) + 1 + return reference_count_map + + @classmethod + def _normalize_file_ids(cls, file_ids: list[str]) -> list[str]: + """校验并标准化文件ID列表。""" + if len(file_ids) > cls.MAX_REFERENCE_FILES: + raise ServiceException(message=f'单个业务对象最多引用{cls.MAX_REFERENCE_FILES}个文件') + try: + normalized_file_ids = [str(uuid.UUID(str(file_id))) for file_id in file_ids] + except (AttributeError, TypeError, ValueError) as exc: + raise ServiceException(message='文件ID格式错误') from exc + return list(dict.fromkeys(normalized_file_ids)) + + @staticmethod + def _normalize_text(value: str | None, field_name: str, max_length: int, required: bool = False) -> str | None: + """校验并标准化业务引用文本字段。""" + normalized_value = value.strip() if value else None + if required and not normalized_value: + raise ServiceException(message=f'{field_name}不能为空') + if normalized_value and (len(normalized_value) > max_length or not normalized_value.isprintable()): + raise ServiceException(message=f'{field_name}格式错误') + return normalized_value + + @staticmethod + def _get_value(item: SysFileInfo | dict, snake_name: str, camel_name: str) -> Any: + """兼容读取ORM对象和查询字典。""" + if isinstance(item, dict): + return item.get(snake_name) if snake_name in item else item.get(camel_name) + return getattr(item, snake_name) + + +class FileRetentionPolicyService: + """ + 文件业务保留策略服务层 + """ + + @classmethod + async def get_file_retention_policy_list_services( + cls, + query_db: AsyncSession, + ) -> list[FileRetentionPolicyModel]: + """ + 获取文件业务保留策略列表service + + :param query_db: orm对象 + :return: 文件业务保留策略列表 + """ + policy_list = await FileRetentionPolicyDao.get_file_retention_policy_list(query_db) + return [FileRetentionPolicyModel(**CamelCaseUtil.transform_result(policy)) for policy in policy_list] + + @classmethod + async def get_enabled_file_retention_policy_services( + cls, + query_db: AsyncSession, + business_type: str, + ) -> FileRetentionPolicyModel | None: + """ + 获取已启用的文件业务保留策略service + + :param query_db: orm对象 + :param business_type: 业务类型 + :return: 文件业务保留策略 + """ + policy = await FileRetentionPolicyDao.get_file_retention_policy_by_business_type( + query_db, + business_type, + enabled_only=True, + ) + return FileRetentionPolicyModel(**CamelCaseUtil.transform_result(policy)) if policy else None + + @classmethod + async def add_file_retention_policy_services( + cls, + query_db: AsyncSession, + policy: FileRetentionPolicyModel, + operator_name: str, + ) -> CrudResponseModel: + """ + 新增文件业务保留策略service + + :param query_db: orm对象 + :param policy: 文件业务保留策略 + :param operator_name: 操作人名称 + :return: 操作结果 + """ + cls._validate_business_type(policy.business_type) + exists_policy = await FileRetentionPolicyDao.get_file_retention_policy_by_business_type( + query_db, + policy.business_type, + ) + if exists_policy: + raise ServiceException(message=f'业务类型{policy.business_type}的保留策略已存在') + current_time = datetime.now() + db_policy = SysFileRetentionPolicy( + **policy.model_dump( + exclude={ + 'create_by', + 'create_time', + 'update_by', + 'update_time', + } + ), + create_by=operator_name, + create_time=current_time, + update_by=operator_name, + update_time=current_time, + ) + try: + await FileRetentionPolicyDao.add_file_retention_policy(query_db, db_policy) + await query_db.commit() + return CrudResponseModel(is_success=True, message='新增成功') + except Exception: + await query_db.rollback() + raise + + @classmethod + async def edit_file_retention_policy_services( + cls, + query_db: AsyncSession, + policy: FileRetentionPolicyModel, + operator_name: str, + ) -> CrudResponseModel: + """ + 修改文件业务保留策略service + + :param query_db: orm对象 + :param policy: 文件业务保留策略 + :param operator_name: 操作人名称 + :return: 操作结果 + """ + cls._validate_business_type(policy.business_type) + exists_policy = await FileRetentionPolicyDao.get_file_retention_policy_by_business_type( + query_db, + policy.business_type, + ) + if exists_policy is None: + raise ServiceException(message='文件业务保留策略不存在') + policy_data = policy.model_dump( + exclude={ + 'business_type', + 'create_by', + 'create_time', + 'update_by', + 'update_time', + } + ) + policy_data.update(update_by=operator_name, update_time=datetime.now()) + try: + await FileRetentionPolicyDao.edit_file_retention_policy( + query_db, + policy.business_type, + policy_data, + ) + await query_db.commit() + return CrudResponseModel(is_success=True, message='修改成功') + except Exception: + await query_db.rollback() + raise + + @classmethod + async def delete_file_retention_policy_services( + cls, + query_db: AsyncSession, + business_type: str, + ) -> CrudResponseModel: + """ + 删除文件业务保留策略service + + :param query_db: orm对象 + :param business_type: 业务类型 + :return: 操作结果 + """ + normalized_business_type = business_type.strip() + cls._validate_business_type(normalized_business_type) + exists_policy = await FileRetentionPolicyDao.get_file_retention_policy_by_business_type( + query_db, + normalized_business_type, + ) + if exists_policy is None: + raise ServiceException(message='文件业务保留策略不存在') + try: + await FileRetentionPolicyDao.delete_file_retention_policy(query_db, normalized_business_type) + await query_db.commit() + return CrudResponseModel(is_success=True, message='删除成功') + except Exception: + await query_db.rollback() + raise + + @staticmethod + def _validate_business_type(business_type: str) -> None: + """校验业务类型。""" + if not business_type.isprintable(): + raise ServiceException(message='业务类型格式错误') + + +class FileRetentionNoticeService: + """ + 文件保留期限提醒服务层 + """ + + DEFAULT_REMIND_DAYS = 7 + MAX_REMIND_DAYS = 365 + MAX_BATCH_SIZE = 1000 + + @classmethod + async def scan_file_retention_notices_services( + cls, + query_db: AsyncSession, + remind_days: int = DEFAULT_REMIND_DAYS, + batch_size: int = 500, + file_data_scope_sql: ColumnElement | None = None, + ) -> FileRetentionScanModel: + """ + 扫描并生成文件保留期限提醒service + + :param query_db: orm对象 + :param remind_days: 提前提醒天数 + :param batch_size: 单类提醒单批处理数量 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :return: 扫描结果 + """ + cls._validate_scan_parameters(remind_days, batch_size) + current_time = datetime.now() + reminder_deadline = current_time + timedelta(days=remind_days) + data_scope_sql = file_data_scope_sql if file_data_scope_sql is not None else true() + try: + expired_files = await FileRetentionNoticeDao.get_missing_notice_candidates( + query_db, + 'expired', + current_time, + reminder_deadline, + batch_size, + data_scope_sql, + ) + expiring_files = await FileRetentionNoticeDao.get_missing_notice_candidates( + query_db, + 'expiring', + current_time, + reminder_deadline, + batch_size, + data_scope_sql, + ) + await FileRetentionNoticeDao.invalidate_expiring_notices( + query_db, + [file_info.file_id for file_info in expired_files], + ) + notice_list = [ + SysFileRetentionNotice( + file_id=file_info.file_id, + notice_type=notice_type, + expire_time=file_info.expire_time, + status='0', + create_time=current_time, + ) + for notice_type, file_infos in ( + ('expired', expired_files), + ('expiring', expiring_files), + ) + for file_info in file_infos + ] + await FileRetentionNoticeDao.add_file_retention_notices(query_db, notice_list) + await query_db.commit() + return FileRetentionScanModel( + expiringCount=len(expiring_files), + expiredCount=len(expired_files), + ) + except Exception: + await query_db.rollback() + raise + + @classmethod + async def get_file_retention_notice_list_services( + cls, + query_db: AsyncSession, + query_object: FileRetentionNoticePageQueryModel, + file_data_scope_sql: ColumnElement, + is_page: bool = True, + ) -> PageModel | list[dict]: + """ + 获取文件保留期限提醒列表service + + :param query_db: orm对象 + :param query_object: 查询参数 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :param is_page: 是否开启分页 + :return: 文件保留期限提醒列表 + """ + return await FileRetentionNoticeDao.get_file_retention_notice_list( + query_db, + query_object, + file_data_scope_sql, + is_page, + ) + + @classmethod + async def mark_file_retention_notices_read_services( + cls, + query_db: AsyncSession, + notice_ids: str, + read_by: str, + file_data_scope_sql: ColumnElement, + ) -> CrudResponseModel: + """ + 标记文件保留期限提醒为已读service + + :param query_db: orm对象 + :param notice_ids: 提醒ID字符串 + :param read_by: 读取者 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :return: 操作结果 + """ + parsed_notice_ids = cls._parse_notice_ids(notice_ids) + scoped_notice_ids = await FileRetentionNoticeDao.get_notice_ids_in_data_scope_for_update( + query_db, + parsed_notice_ids, + file_data_scope_sql, + ) + if len(scoped_notice_ids) != len(parsed_notice_ids): + await query_db.rollback() + raise ServiceException(message='部分提醒不存在、已失效或超出数据权限') + try: + await FileRetentionNoticeDao.mark_file_retention_notices_read( + query_db, + parsed_notice_ids, + read_by, + datetime.now(), + ) + await query_db.commit() + return CrudResponseModel(is_success=True, message='提醒已标记为已读') + except Exception: + await query_db.rollback() + raise + + @classmethod + def _validate_scan_parameters(cls, remind_days: int, batch_size: int) -> None: + """校验提醒扫描参数。""" + if remind_days < 1 or remind_days > cls.MAX_REMIND_DAYS: + raise ServiceException(message=f'提前提醒天数必须在1到{cls.MAX_REMIND_DAYS}之间') + if batch_size < 1 or batch_size > cls.MAX_BATCH_SIZE: + raise ServiceException(message=f'单批处理数量必须在1到{cls.MAX_BATCH_SIZE}之间') + + @staticmethod + def _parse_notice_ids(notice_ids: str) -> list[int]: + """解析并校验提醒ID。""" + try: + parsed_notice_ids = list(dict.fromkeys(int(item.strip()) for item in notice_ids.split(',') if item.strip())) + except (AttributeError, ValueError) as exc: + raise ServiceException(message='提醒ID格式不正确') from exc + if not parsed_notice_ids or any(notice_id <= 0 for notice_id in parsed_notice_ids): + raise ServiceException(message='提醒ID格式不正确') + return parsed_notice_ids diff --git a/ruoyi-fastapi-backend/module_admin/service/file_service.py b/ruoyi-fastapi-backend/module_admin/service/file_service.py new file mode 100644 index 0000000..65bf281 --- /dev/null +++ b/ruoyi-fastapi-backend/module_admin/service/file_service.py @@ -0,0 +1,1654 @@ +import asyncio +import mimetypes +import uuid +from dataclasses import asdict, dataclass +from datetime import datetime, timedelta +from typing import Any, Literal + +from fastapi import Request +from sqlalchemy import ColumnElement +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from common.vo import CrudResponseModel, PageModel +from config.database import AsyncSessionLocal +from config.env import UploadConfig +from exceptions.exception import ServiceException +from module_admin.dao.file_access_dao import FileAccessLogDao +from module_admin.dao.file_business_dao import FileReferenceDao, FileRetentionNoticeDao +from module_admin.dao.file_info_dao import FileInfoDao +from module_admin.entity.do.file_do import SysFileInfo, SysFileReconcileIssue, SysFileReconcileRun, SysFileReference +from module_admin.entity.vo.file_vo import ( + DeleteFileModel, + DisposeExpiredFileModel, + ExtendFileRetentionModel, + FileAccessLogPageQueryModel, + FileInfoDisplayModel, + FileInfoModel, + FileInfoPageQueryModel, + FileReconcileAction, + FileReconcileHandleModel, + FileReconcileIssueModel, + FileReconcileIssuePageQueryModel, + FileReconcileRunModel, + FileReconcileRunPageQueryModel, + FileReconcileStatsModel, + FileStatsModel, + TransferFileModel, +) +from module_admin.entity.vo.user_vo import CurrentUserModel +from module_admin.service.file_access_service import FileAuditService +from module_admin.service.file_business_service import FileReferenceService +from utils.file_util import FileReconcileUtil, FileUtil +from utils.log_util import logger +from utils.upload_util import UploadUtil + + +@dataclass(frozen=True) +class FileAuditSnapshot: + """文件审计快照。""" + + file_id: str + original_name: str + access_type: str + + +class FileRetentionDispositionService: + """ + 文件到期处置服务层 + """ + + @classmethod + async def extend_file_retention_services( + cls, + query_db: AsyncSession, + current_user: CurrentUserModel, + notice_id: int, + extend_retention: ExtendFileRetentionModel, + file_data_scope_sql: ColumnElement, + request: Request | None = None, + ) -> CrudResponseModel: + """ + 延长文件保留期限service + + :param query_db: orm对象 + :param current_user: 当前用户对象 + :param notice_id: 提醒ID + :param extend_retention: 延期参数 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :param request: Request对象 + :return: 延期结果 + """ + user = current_user.user + if user is None or not user.user_name: + raise ServiceException(message='无法获取当前用户信息') + context = await FileRetentionNoticeDao.get_file_retention_notice_context_for_update( + query_db, + notice_id, + file_data_scope_sql, + ) + if context is None: + await query_db.rollback() + raise ServiceException(message='提醒不存在、已失效或超出数据权限') + _, file_info = context + file_snapshot = FileAuditSnapshot( + file_id=file_info.file_id, + original_name=file_info.original_name, + access_type=file_info.access_type, + ) + current_time = datetime.now() + new_expire_time = extend_retention.expire_time + if new_expire_time.tzinfo: + new_expire_time = new_expire_time.astimezone().replace(tzinfo=None) + previous_expire_time = file_info.expire_time + if previous_expire_time is None: + await query_db.rollback() + raise ServiceException(message='文件未配置保留期限') + if new_expire_time <= current_time or new_expire_time <= previous_expire_time: + await cls._enqueue_retention_audit( + request, + current_user, + file_snapshot, + 'retention_extend', + 'denied', + extend_retention.reason, + previous_expire_time, + new_expire_time, + error_message='InvalidExpireTime', + ) + await query_db.rollback() + raise ServiceException(message='新的到期时间必须晚于当前时间和原到期时间') + + reference_list = await FileReferenceDao.get_file_reference_list_for_update(query_db, file_info.file_id) + if not reference_list: + await cls._enqueue_retention_audit( + request, + current_user, + file_snapshot, + 'retention_extend', + 'denied', + extend_retention.reason, + previous_expire_time, + new_expire_time, + error_message='TimedBusinessReferenceNotFound', + ) + await query_db.rollback() + raise ServiceException(message='文件不存在可延期的限时业务引用，请先重新关联业务') + if (file_info.business_type and file_info.business_id) or any( + reference.retention_expire_time is None for reference in reference_list + ): + await cls._enqueue_retention_audit( + request, + current_user, + file_snapshot, + 'retention_extend', + 'denied', + extend_retention.reason, + previous_expire_time, + new_expire_time, + error_message='PermanentBusinessReferenceExists', + ) + await query_db.rollback() + raise ServiceException(message='文件存在永久业务引用，不能通过到期提醒延期') + + terminal_references = [ + reference for reference in reference_list if reference.retention_expire_time == previous_expire_time + ] + if reference_list and not terminal_references: + await query_db.rollback() + raise ServiceException(message='文件到期时间与业务引用不一致，请先检查业务引用') + + for reference in terminal_references: + reference.retention_expire_time = new_expire_time + file_info.expire_time = new_expire_time + file_info.update_by = user.user_name + file_info.update_time = current_time + await FileRetentionNoticeDao.invalidate_file_retention_notices(query_db, file_info.file_id) + try: + await query_db.commit() + except Exception as exc: + await query_db.rollback() + await cls._enqueue_retention_audit( + request, + current_user, + file_snapshot, + 'retention_extend', + 'failed', + extend_retention.reason, + previous_expire_time, + new_expire_time, + reference_count=len(terminal_references), + error_message=exc.__class__.__name__, + ) + raise + + await cls._enqueue_retention_audit( + request, + current_user, + file_snapshot, + 'retention_extend', + 'completed', + extend_retention.reason, + previous_expire_time, + new_expire_time, + reference_count=len(terminal_references), + ) + return CrudResponseModel(is_success=True, message='文件保留期限已延长') + + @classmethod + async def dispose_expired_file_services( + cls, + query_db: AsyncSession, + current_user: CurrentUserModel, + notice_id: int, + dispose_file: DisposeExpiredFileModel, + file_data_scope_sql: ColumnElement, + request: Request | None = None, + ) -> CrudResponseModel: + """ + 将到期文件移入回收站service + + :param query_db: orm对象 + :param current_user: 当前用户对象 + :param notice_id: 提醒ID + :param dispose_file: 处置参数 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :param request: Request对象 + :return: 处置结果 + """ + user = current_user.user + if user is None or not user.user_name: + raise ServiceException(message='无法获取当前用户信息') + context = await FileRetentionNoticeDao.get_file_retention_notice_context_for_update( + query_db, + notice_id, + file_data_scope_sql, + ) + if context is None: + await query_db.rollback() + raise ServiceException(message='提醒不存在、已失效或超出数据权限') + _, file_info = context + file_snapshot = FileAuditSnapshot( + file_id=file_info.file_id, + original_name=file_info.original_name, + access_type=file_info.access_type, + ) + current_time = datetime.now() + expire_time = file_info.expire_time + if expire_time is None or expire_time > current_time: + await cls._enqueue_retention_audit( + request, + current_user, + file_snapshot, + 'retention_dispose', + 'denied', + dispose_file.reason, + expire_time, + error_message='FileNotExpired', + ) + await query_db.rollback() + raise ServiceException(message='文件尚未到期，不能执行到期处置') + + reference_list = await FileReferenceDao.get_file_reference_list_for_update(query_db, file_info.file_id) + blocking_references = [ + reference + for reference in reference_list + if reference.retention_expire_time is None or reference.retention_expire_time > current_time + ] + if (file_info.business_type and file_info.business_id) or blocking_references: + await cls._enqueue_retention_audit( + request, + current_user, + file_snapshot, + 'retention_dispose', + 'denied', + dispose_file.reason, + expire_time, + reference_count=len(reference_list), + error_message='ActiveBusinessReferenceExists', + ) + await query_db.rollback() + raise ServiceException(message='文件存在永久或尚未到期的业务引用，不能执行到期处置') + + reference_snapshots = cls._build_reference_snapshots(reference_list) + try: + staged_files = await asyncio.to_thread(FileUtil.stage_file_deletions, [file_info]) + except (OSError, ValueError) as exc: + await query_db.rollback() + await cls._enqueue_retention_audit( + request, + current_user, + file_snapshot, + 'retention_dispose', + 'failed', + dispose_file.reason, + expire_time, + reference_count=len(reference_list), + error_message=exc.__class__.__name__, + ) + raise ServiceException(message='到期文件移入回收区失败') from exc + + try: + await FileReferenceDao.delete_file_references(query_db, file_info.file_id) + await FileInfoDao.soft_delete_file_infos( + query_db, + [file_info.file_id], + user.user_name, + current_time, + ) + await query_db.commit() + except Exception as exc: + await query_db.rollback() + await asyncio.to_thread(FileUtil.restore_staged_files, staged_files) + await cls._enqueue_retention_audit( + request, + current_user, + file_snapshot, + 'retention_dispose', + 'failed', + dispose_file.reason, + expire_time, + reference_count=len(reference_list), + error_message=exc.__class__.__name__, + ) + raise + + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_snapshot.file_id, + 'retention_dispose', + 'completed', + operation_detail={ + 'originalName': file_snapshot.original_name, + 'accessType': file_snapshot.access_type, + 'reason': dispose_file.reason, + 'expireTime': expire_time, + 'releasedReferenceCount': len(reference_snapshots), + 'releasedReferences': reference_snapshots, + 'previousStatus': 'active', + 'newStatus': 'deleted', + }, + ) + return CrudResponseModel(is_success=True, message='到期文件已移入回收站') + + @classmethod + async def _enqueue_retention_audit( + cls, + request: Request | None, + current_user: CurrentUserModel, + file_snapshot: FileAuditSnapshot, + action: Literal['retention_extend', 'retention_dispose'], + result: Literal['denied', 'completed', 'failed'], + reason: str, + previous_expire_time: datetime | None, + new_expire_time: datetime | None = None, + reference_count: int | None = None, + error_message: str = '', + ) -> None: + """写入文件到期处置审计。""" + operation_detail = { + 'originalName': file_snapshot.original_name, + 'accessType': file_snapshot.access_type, + 'reason': reason, + 'previousExpireTime': previous_expire_time, + } + if new_expire_time is not None: + operation_detail['newExpireTime'] = new_expire_time + if reference_count is not None: + operation_detail['referenceCount'] = reference_count + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_snapshot.file_id, + action, + result, + error_message=error_message, + operation_detail=operation_detail, + ) + + @staticmethod + def _build_reference_snapshots(reference_list: list[SysFileReference]) -> list[dict[str, Any]]: + """构建释放业务引用的审计快照。""" + return [ + { + 'referenceId': reference.reference_id, + 'businessType': reference.business_type, + 'businessId': reference.business_id, + 'businessName': reference.business_name, + 'retentionExpireTime': reference.retention_expire_time, + } + for reference in reference_list + ] + + +class FileLifecycleService: + MAX_PURGE_RETENTION_DAYS = 36500 + MAX_PURGE_BATCH_SIZE = 1000 + + @classmethod + async def delete_file_services( + cls, + query_db: AsyncSession, + current_user: CurrentUserModel, + delete_file: DeleteFileModel, + file_data_scope_sql: ColumnElement, + request: Request | None = None, + ) -> CrudResponseModel: + """ + 删除文件service + + :param query_db: orm对象 + :param current_user: 当前用户对象 + :param delete_file: 删除文件参数 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :param request: Request对象 + :return: 删除结果 + """ + file_ids = FileUtil.parse_file_ids(delete_file.file_ids) + user = current_user.user + if user is None or not user.user_name: + raise ServiceException(message='无法获取当前用户信息') + + file_infos = await FileInfoDao.get_file_infos_by_ids_for_update(query_db, file_ids, file_data_scope_sql) + if len(file_infos) != len(file_ids): + await query_db.rollback() + raise ServiceException(message='部分文件不存在、已删除或超出数据权限') + file_audit_snapshots = cls._build_file_audit_snapshots(file_infos) + reference_count_map = await FileReferenceService.get_file_reference_count_map_services(query_db, file_infos) + referenced_file_ids = [file_id for file_id in file_ids if reference_count_map.get(file_id, 0) > 0] + if referenced_file_ids: + await query_db.rollback() + snapshot_map = {item.file_id: item for item in file_audit_snapshots} + for file_audit_snapshot in file_audit_snapshots: + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_audit_snapshot.file_id, + 'delete', + 'denied', + error_message='BusinessReferenceExists', + operation_detail={ + 'originalName': file_audit_snapshot.original_name, + 'accessType': file_audit_snapshot.access_type, + 'referenceCount': reference_count_map.get(file_audit_snapshot.file_id, 0), + }, + ) + referenced_names = [snapshot_map[file_id].original_name for file_id in referenced_file_ids[:3]] + referenced_name_text = '、'.join(f'“{name}”' for name in referenced_names) + if len(referenced_file_ids) > len(referenced_names): + referenced_name_text += f'等{len(referenced_file_ids)}个文件' + raise ServiceException(message=f'文件{referenced_name_text}仍被业务引用，请先解除引用后再删除') + + try: + staged_files = await asyncio.to_thread(FileUtil.stage_file_deletions, file_infos) + except (OSError, ValueError) as exc: + await query_db.rollback() + for file_audit_snapshot in file_audit_snapshots: + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_audit_snapshot.file_id, + 'delete', + 'failed', + error_message=exc.__class__.__name__, + operation_detail={ + 'originalName': file_audit_snapshot.original_name, + 'accessType': file_audit_snapshot.access_type, + }, + ) + raise ServiceException(message='文件移入回收区失败') from exc + + try: + await FileInfoDao.soft_delete_file_infos(query_db, file_ids, user.user_name, datetime.now()) + await query_db.commit() + except Exception as exc: + await query_db.rollback() + await asyncio.to_thread(FileUtil.restore_staged_files, staged_files) + for file_audit_snapshot in file_audit_snapshots: + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_audit_snapshot.file_id, + 'delete', + 'failed', + error_message=exc.__class__.__name__, + operation_detail={ + 'originalName': file_audit_snapshot.original_name, + 'accessType': file_audit_snapshot.access_type, + }, + ) + raise + + for file_audit_snapshot in file_audit_snapshots: + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_audit_snapshot.file_id, + 'delete', + 'completed', + operation_detail={ + 'originalName': file_audit_snapshot.original_name, + 'accessType': file_audit_snapshot.access_type, + 'previousStatus': 'active', + 'newStatus': 'deleted', + }, + ) + return CrudResponseModel(is_success=True, message='文件已移入回收站') + + @classmethod + async def purge_file_services( + cls, + query_db: AsyncSession, + current_user: CurrentUserModel, + file_ids: str, + file_data_scope_sql: ColumnElement, + request: Request | None = None, + ) -> CrudResponseModel: + """ + 永久清理回收站文件service + + :param query_db: orm对象 + :param current_user: 当前用户对象 + :param file_ids: 文件ID字符串 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :param request: Request对象 + :return: 永久清理结果 + """ + parsed_file_ids = FileUtil.parse_file_ids(file_ids) + user = current_user.user + if user is None or not user.user_name: + raise ServiceException(message='无法获取当前用户信息') + file_infos = await FileInfoDao.get_purgeable_file_infos_by_ids_for_update( + query_db, + parsed_file_ids, + file_data_scope_sql, + ) + if len(file_infos) != len(parsed_file_ids): + await query_db.rollback() + raise ServiceException(message='部分文件不存在、未进入回收站或超出数据权限') + file_audit_snapshots = cls._build_file_audit_snapshots(file_infos) + reference_count_map = await FileReferenceService.get_file_reference_count_map_services(query_db, file_infos) + if any(reference_count_map.get(file_id, 0) > 0 for file_id in parsed_file_ids): + await query_db.rollback() + for file_audit_snapshot in file_audit_snapshots: + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_audit_snapshot.file_id, + 'purge', + 'denied', + error_message='BusinessReferenceExists', + operation_detail={ + 'originalName': file_audit_snapshot.original_name, + 'accessType': file_audit_snapshot.access_type, + 'referenceCount': reference_count_map.get(file_audit_snapshot.file_id, 0), + }, + ) + raise ServiceException(message='部分文件仍被业务引用，不能永久清理') + try: + staged_files = await asyncio.to_thread(FileUtil.prepare_deleted_files_for_purge, file_infos) + except (OSError, ValueError) as exc: + await query_db.rollback() + await cls._enqueue_purge_audits( + request, + current_user, + file_audit_snapshots, + 'failed', + error_message=exc.__class__.__name__, + ) + raise ServiceException(message='回收区文件校验失败，未执行永久清理') from exc + + try: + await FileInfoDao.mark_file_infos_purging( + query_db, + parsed_file_ids, + user.user_name, + datetime.now(), + ) + await query_db.commit() + except Exception: + await query_db.rollback() + raise + + try: + await asyncio.to_thread(FileUtil.purge_deleted_files, staged_files) + except (OSError, ValueError) as exc: + await cls._enqueue_purge_audits( + request, + current_user, + file_audit_snapshots, + 'failed', + error_message=exc.__class__.__name__, + ) + raise ServiceException(message='永久清理物理文件失败，文件已保留为清理中状态，可重试') from exc + + try: + await FileInfoDao.purge_file_infos(query_db, parsed_file_ids) + await query_db.commit() + except Exception as exc: + await query_db.rollback() + await cls._enqueue_purge_audits( + request, + current_user, + file_audit_snapshots, + 'failed', + error_message=exc.__class__.__name__, + ) + raise + + await cls._enqueue_purge_audits( + request, + current_user, + file_audit_snapshots, + 'completed', + ) + return CrudResponseModel(is_success=True, message='文件已永久清理') + + @classmethod + async def purge_recycle_bin_services( + cls, + query_db: AsyncSession, + retention_days: int = 30, + batch_size: int = 100, + ) -> int: + """ + 按回收站保留天数自动永久清理文件service + + :param query_db: orm对象 + :param retention_days: 回收站保留天数 + :param batch_size: 单批处理数量 + :return: 永久清理文件数量 + """ + cls._validate_purge_parameters(retention_days, batch_size) + deleted_before = datetime.now() - timedelta(days=retention_days) + file_infos = await FileInfoDao.get_recycle_bin_purge_candidates( + query_db, + deleted_before, + batch_size, + ) + if not file_infos: + await query_db.rollback() + return 0 + file_ids = [file_info.file_id for file_info in file_infos] + file_audit_snapshots = cls._build_file_audit_snapshots(file_infos) + reference_count_map = await FileReferenceService.get_file_reference_count_map_services(query_db, file_infos) + if any(reference_count_map.get(file_id, 0) > 0 for file_id in file_ids): + await query_db.rollback() + raise ServiceException(message='自动清理候选文件仍存在业务引用') + try: + staged_files = await asyncio.to_thread(FileUtil.prepare_deleted_files_for_purge, file_infos) + await FileInfoDao.mark_file_infos_purging(query_db, file_ids, 'system', datetime.now()) + await query_db.commit() + except Exception: + await query_db.rollback() + raise + + try: + await asyncio.to_thread(FileUtil.purge_deleted_files, staged_files) + except (OSError, ValueError) as exc: + await query_db.rollback() + for file_audit_snapshot in file_audit_snapshots: + await FileAuditService.add_system_file_audit( + query_db, + file_audit_snapshot.file_id, + 'purge', + 'failed', + error_message=exc.__class__.__name__, + operation_detail={ + 'originalName': file_audit_snapshot.original_name, + 'accessType': file_audit_snapshot.access_type, + 'automatic': True, + 'retentionDays': retention_days, + }, + ) + await query_db.commit() + raise + + try: + for file_audit_snapshot in file_audit_snapshots: + await FileAuditService.add_system_file_audit( + query_db, + file_audit_snapshot.file_id, + 'purge', + 'completed', + operation_detail={ + 'originalName': file_audit_snapshot.original_name, + 'accessType': file_audit_snapshot.access_type, + 'automatic': True, + 'retentionDays': retention_days, + }, + ) + await FileInfoDao.purge_file_infos(query_db, file_ids) + await query_db.commit() + except Exception: + await query_db.rollback() + raise + return len(file_ids) + + @classmethod + async def restore_file_services( + cls, + query_db: AsyncSession, + current_user: CurrentUserModel, + file_ids: str, + file_data_scope_sql: ColumnElement, + request: Request | None = None, + ) -> CrudResponseModel: + """ + 恢复文件service + + :param query_db: orm对象 + :param current_user: 当前用户对象 + :param file_ids: 文件ID字符串 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :param request: Request对象 + :return: 恢复结果 + """ + parsed_file_ids = FileUtil.parse_file_ids(file_ids) + user = current_user.user + if user is None or not user.user_name: + raise ServiceException(message='无法获取当前用户信息') + + file_infos = await FileInfoDao.get_deleted_file_infos_by_ids_for_update( + query_db, + parsed_file_ids, + file_data_scope_sql, + ) + if len(file_infos) != len(parsed_file_ids): + await query_db.rollback() + raise ServiceException(message='部分文件不存在、未删除或超出数据权限') + file_audit_snapshots = cls._build_file_audit_snapshots(file_infos) + + try: + staged_files = await asyncio.to_thread(FileUtil.prepare_deleted_files_for_restore, file_infos) + except (OSError, ValueError) as exc: + await query_db.rollback() + for file_audit_snapshot in file_audit_snapshots: + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_audit_snapshot.file_id, + 'restore', + 'failed', + error_message=exc.__class__.__name__, + operation_detail={ + 'originalName': file_audit_snapshot.original_name, + 'accessType': file_audit_snapshot.access_type, + }, + ) + raise ServiceException(message='文件从回收区恢复失败') from exc + + try: + await FileInfoDao.restore_file_infos( + query_db, + parsed_file_ids, + user.user_name, + datetime.now(), + ) + await query_db.commit() + except Exception as exc: + await query_db.rollback() + for file_audit_snapshot in file_audit_snapshots: + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_audit_snapshot.file_id, + 'restore', + 'failed', + error_message=exc.__class__.__name__, + operation_detail={ + 'originalName': file_audit_snapshot.original_name, + 'accessType': file_audit_snapshot.access_type, + }, + ) + raise + + try: + await asyncio.to_thread(FileUtil.restore_deleted_files, staged_files) + except (OSError, ValueError) as exc: + try: + await FileInfoDao.soft_delete_file_infos( + query_db, + parsed_file_ids, + user.user_name, + datetime.now(), + ) + await query_db.commit() + except Exception as compensation_exc: + await query_db.rollback() + logger.error(f'文件恢复状态补偿失败: {compensation_exc}') + for file_audit_snapshot in file_audit_snapshots: + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_audit_snapshot.file_id, + 'restore', + 'failed', + error_message=exc.__class__.__name__, + operation_detail={ + 'originalName': file_audit_snapshot.original_name, + 'accessType': file_audit_snapshot.access_type, + }, + ) + raise ServiceException(message='文件从回收区恢复失败') from exc + + await asyncio.to_thread(FileUtil.cleanup_trash_directories, staged_files) + for file_audit_snapshot in file_audit_snapshots: + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_audit_snapshot.file_id, + 'restore', + 'completed', + operation_detail={ + 'originalName': file_audit_snapshot.original_name, + 'accessType': file_audit_snapshot.access_type, + 'previousStatus': 'deleted', + 'newStatus': 'active', + }, + ) + return CrudResponseModel(is_success=True, message='文件恢复成功') + + @staticmethod + def _build_file_audit_snapshots(file_infos: list[SysFileInfo]) -> list[FileAuditSnapshot]: + """ + 在事务结束前固化文件审计字段 + + :param file_infos: 文件信息列表 + :return: 文件审计快照列表 + """ + return [ + FileAuditSnapshot( + file_id=file_info.file_id, + original_name=file_info.original_name, + access_type=file_info.access_type, + ) + for file_info in file_infos + ] + + @classmethod + async def _enqueue_purge_audits( + cls, + request: Request | None, + current_user: CurrentUserModel, + file_audit_snapshots: list[FileAuditSnapshot], + result: str, + error_message: str = '', + ) -> None: + """批量写入永久清理审计。""" + for file_audit_snapshot in file_audit_snapshots: + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_audit_snapshot.file_id, + 'purge', + result, + error_message=error_message, + operation_detail={ + 'originalName': file_audit_snapshot.original_name, + 'accessType': file_audit_snapshot.access_type, + 'automatic': False, + }, + ) + + @classmethod + def _validate_purge_parameters(cls, retention_days: int, batch_size: int) -> None: + """校验自动清理参数。""" + if retention_days < 1 or retention_days > cls.MAX_PURGE_RETENTION_DAYS: + raise ServiceException(message=f'回收站保留天数必须在1到{cls.MAX_PURGE_RETENTION_DAYS}之间') + if batch_size < 1 or batch_size > cls.MAX_PURGE_BATCH_SIZE: + raise ServiceException(message=f'单批处理数量必须在1到{cls.MAX_PURGE_BATCH_SIZE}之间') + + +class FileQueryService: + @classmethod + async def get_file_list_services( + cls, + query_db: AsyncSession, + query_object: FileInfoPageQueryModel, + file_data_scope_sql: ColumnElement, + is_page: bool = True, + ) -> PageModel | list[dict[str, Any]]: + """ + 获取文件信息列表service + + :param query_db: orm对象 + :param query_object: 文件信息查询参数 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :param is_page: 是否开启分页 + :return: 文件信息列表 + """ + file_list = await FileInfoDao.get_file_info_list(query_db, query_object, file_data_scope_sql, is_page) + file_rows = file_list.rows if isinstance(file_list, PageModel) else file_list + reference_count_map = await FileReferenceService.get_file_reference_count_map_services(query_db, file_rows) + cls._enrich_file_reference_counts(file_rows, reference_count_map) + await asyncio.to_thread(FileUtil.enrich_storage_status, file_rows) + return file_list + + @classmethod + async def get_file_stats_services( + cls, + query_db: AsyncSession, + query_object: FileInfoPageQueryModel, + file_data_scope_sql: ColumnElement, + ) -> FileStatsModel: + """ + 获取文件管理统计信息service + + :param query_db: orm对象 + :param query_object: 文件信息查询参数 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :return: 文件管理统计信息 + """ + return await FileInfoDao.get_file_stats(query_db, query_object, file_data_scope_sql) + + @classmethod + async def file_detail_services( + cls, + query_db: AsyncSession, + file_id: str, + file_data_scope_sql: ColumnElement, + ) -> FileInfoDisplayModel: + """ + 获取文件详细信息service + + :param query_db: orm对象 + :param file_id: 文件ID + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :return: 文件详细信息 + """ + file_info = await FileInfoDao.get_file_management_detail_by_id(query_db, file_id, file_data_scope_sql) + if file_info is None: + raise ServiceException(message='文件信息不存在或超出数据权限') + reference_count_map = await FileReferenceService.get_file_reference_count_map_services(query_db, [file_info]) + file_info['reference_count'] = reference_count_map.get(file_id, 0) + file_info['storage_status'] = await asyncio.to_thread(FileUtil.get_storage_status, file_info) + return FileInfoDisplayModel.model_validate(file_info, by_name=True) + + @classmethod + async def get_file_access_log_list_services( + cls, + query_db: AsyncSession, + file_id: str, + query_object: FileAccessLogPageQueryModel, + file_data_scope_sql: ColumnElement, + is_page: bool = True, + ) -> PageModel | list[dict[str, Any]]: + """ + 获取文件访问审计列表service + + :param query_db: orm对象 + :param file_id: 文件ID + :param query_object: 文件访问审计查询参数 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :param is_page: 是否开启分页 + :return: 文件访问审计列表 + """ + if await FileInfoDao.get_file_info_detail_by_id(query_db, file_id, file_data_scope_sql) is None: + raise ServiceException(message='文件信息不存在或超出数据权限') + return await FileAccessLogDao.get_file_access_log_list(query_db, file_id, query_object, is_page) + + @staticmethod + def _enrich_file_reference_counts(file_rows: list[dict[str, Any]], reference_count_map: dict[str, int]) -> None: + """ + 补充文件业务引用数量 + + :param file_rows: 文件信息列表 + :param reference_count_map: 文件ID和业务引用数量映射 + :return: None + """ + for file_row in file_rows: + file_id = str(file_row.get('file_id') or file_row.get('fileId')) + file_row['referenceCount'] = reference_count_map.get(file_id, 0) + + +class FileTransferService: + @classmethod + async def transfer_file_services( + cls, + query_db: AsyncSession, + current_user: CurrentUserModel, + file_ids: str, + transfer_file: TransferFileModel, + file_data_scope_sql: ColumnElement, + user_data_scope_sql: ColumnElement, + dept_data_scope_sql: ColumnElement, + request: Request | None = None, + ) -> CrudResponseModel: + """ + 批量转移文件service + + :param query_db: orm对象 + :param current_user: 当前用户对象 + :param file_ids: 文件ID字符串 + :param transfer_file: 文件转移参数 + :param file_data_scope_sql: 文件数据权限对应的查询sql语句 + :param user_data_scope_sql: 用户数据权限对应的查询sql语句 + :param dept_data_scope_sql: 部门数据权限对应的查询sql语句 + :param request: Request对象 + :return: 转移结果 + """ + parsed_file_ids = FileUtil.parse_file_ids(file_ids) + user = current_user.user + if user is None or not user.user_name: + raise ServiceException(message='无法获取当前用户信息') + + target_user = await FileInfoDao.get_transfer_user_by_id( + query_db, + transfer_file.owner_user_id, + user_data_scope_sql, + ) + target_dept = await FileInfoDao.get_transfer_dept_by_id( + query_db, + transfer_file.dept_id, + dept_data_scope_sql, + ) + if target_user is None or target_dept is None: + await query_db.rollback() + raise ServiceException(message='目标用户或部门不存在、已停用或超出数据权限') + if target_user.dept_id != target_dept.dept_id: + await query_db.rollback() + raise ServiceException(message='目标用户不属于所选部门') + target_user_id = target_user.user_id + target_user_name = getattr(target_user, 'user_name', '') + target_dept_id = target_dept.dept_id + + file_infos = await FileInfoDao.get_file_infos_by_ids_for_update( + query_db, + parsed_file_ids, + file_data_scope_sql, + ) + if len(file_infos) != len(parsed_file_ids): + await query_db.rollback() + raise ServiceException(message='部分文件不存在、已删除或超出数据权限') + + ownership_snapshots = { + file_info.file_id: { + 'previousOwnerUserId': getattr(file_info, 'owner_user_id', None), + 'previousDeptId': getattr(file_info, 'dept_id', None), + 'previousUploaderAccessEnabled': getattr(file_info, 'uploader_access_enabled', '1') in {'1', True}, + } + for file_info in file_infos + } + + try: + await FileInfoDao.transfer_file_infos( + query_db, + parsed_file_ids, + target_user_id, + target_dept_id, + transfer_file.retain_uploader_access, + user.user_name, + datetime.now(), + ) + await query_db.commit() + except Exception as exc: + await query_db.rollback() + for file_id in parsed_file_ids: + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_id, + 'transfer', + 'failed', + error_message=exc.__class__.__name__, + operation_detail={ + **ownership_snapshots[file_id], + 'newOwnerUserId': target_user_id, + 'newDeptId': target_dept_id, + 'newUploaderAccessEnabled': transfer_file.retain_uploader_access, + 'reason': transfer_file.reason, + }, + ) + raise + for file_id in parsed_file_ids: + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_id, + 'transfer', + 'completed', + operation_detail={ + **ownership_snapshots[file_id], + 'newOwnerUserId': target_user_id, + 'newOwnerName': target_user_name, + 'newDeptId': target_dept_id, + 'newUploaderAccessEnabled': transfer_file.retain_uploader_access, + 'reason': transfer_file.reason, + }, + ) + return CrudResponseModel(is_success=True, message='文件转移成功') + + +class FileReconcileService: + """ + 文件存储对账服务 + """ + + RUN_LOCK_NAME = 'storage_reconcile' + RUN_STALE_HOURS = 6 + ISSUE_ACTIONS: dict[str, list[FileReconcileAction]] = { + 'unexpected_trash': ['restore_source'], + 'unexpected_source': ['move_to_trash'], + 'wrong_storage_root': ['move_to_expected_root'], + 'duplicate_file': ['quarantine_file'], + 'orphan_file': ['quarantine_file', 'register_orphan'], + 'size_mismatch': ['accept_current'], + 'hash_mismatch': ['accept_current'], + } + + @classmethod + async def start_reconcile_run_services( + cls, + query_db: AsyncSession, + *, + check_hash: bool = False, + trigger_type: str = 'manual', + current_user: CurrentUserModel | None = None, + ) -> FileReconcileRunModel: + """ + 创建文件存储对账任务 + + :param query_db: orm对象 + :param check_hash: 是否校验文件摘要 + :param trigger_type: 触发类型 + :param current_user: 当前用户对象 + :return: 对账任务 + """ + if trigger_type not in {'manual', 'scheduled'}: + raise ServiceException(message='对账任务触发类型不正确') + started_by = 'system' + if trigger_type == 'manual': + user = cls._require_admin(current_user) + started_by = user.user_name + current_time = datetime.now() + reconcile_run = SysFileReconcileRun( + run_id=str(uuid.uuid4()), + trigger_type=trigger_type, + status='running', + check_hash='1' if check_hash else '0', + lock_name=cls.RUN_LOCK_NAME, + started_by=started_by, + started_time=current_time, + ) + run_data = cls._build_run_data(reconcile_run) + try: + await FileInfoDao.release_stale_runs( + query_db, + current_time - timedelta(hours=cls.RUN_STALE_HOURS), + current_time, + ) + await FileInfoDao.add_reconcile_run(query_db, reconcile_run) + await query_db.commit() + except IntegrityError as exc: + await query_db.rollback() + raise ServiceException(message='已有文件存储对账任务正在运行') from exc + except Exception: + await query_db.rollback() + raise + return FileReconcileRunModel.model_validate(run_data, by_name=True) + + @classmethod + async def execute_reconcile_run_services(cls, run_id: str) -> None: + """ + 在独立会话中执行文件存储对账任务 + + :param run_id: 任务ID + :return: None + """ + try: + async with AsyncSessionLocal() as query_db: + reconcile_run = await FileInfoDao.get_reconcile_run_by_id(query_db, run_id) + if reconcile_run is None or reconcile_run.status != 'running': + return + check_hash = reconcile_run.check_hash == '1' + file_infos = await FileInfoDao.get_all_local_file_infos(query_db) + scan_result = await asyncio.to_thread( + FileReconcileUtil.scan_storage, + file_infos, + check_hash, + ) + current_time = datetime.now() + new_issue_count = await FileInfoDao.upsert_reconcile_issues( + query_db, + run_id, + [asdict(finding) for finding in scan_result.findings], + current_time, + ) + resolved_issue_count = await FileInfoDao.resolve_disappeared_issues( + query_db, + run_id, + current_time, + ) + await FileInfoDao.finish_reconcile_run( + query_db, + run_id, + status='completed', + finished_time=current_time, + scanned_file_count=scan_result.scanned_file_count, + scanned_storage_count=scan_result.scanned_storage_count, + issue_count=len(scan_result.findings), + new_issue_count=new_issue_count, + resolved_issue_count=resolved_issue_count, + ) + await query_db.commit() + logger.info( + f'文件存储对账任务{run_id}完成，扫描文件记录{scan_result.scanned_file_count}条，' + f'物理文件{scan_result.scanned_storage_count}个，发现异常{len(scan_result.findings)}个' + ) + except Exception as exc: + logger.exception(f'文件存储对账任务{run_id}执行失败') + async with AsyncSessionLocal() as query_db: + try: + await FileInfoDao.finish_reconcile_run( + query_db, + run_id, + status='failed', + finished_time=datetime.now(), + error_message=f'{exc.__class__.__name__}：对账任务执行失败', + ) + await query_db.commit() + except Exception: + await query_db.rollback() + logger.exception(f'文件存储对账任务{run_id}失败状态更新失败') + + @classmethod + async def run_scheduled_reconcile_services(cls, check_hash: bool = False) -> None: + """ + 执行定时文件存储对账 + + :param check_hash: 是否校验文件摘要 + :return: None + """ + async with AsyncSessionLocal() as query_db: + try: + reconcile_run = await cls.start_reconcile_run_services( + query_db, + check_hash=check_hash, + trigger_type='scheduled', + ) + except ServiceException as exc: + logger.warning(f'定时文件存储对账未启动：{exc.message}') + return + await cls.execute_reconcile_run_services(reconcile_run.run_id) + + @classmethod + async def get_reconcile_run_list_services( + cls, + query_db: AsyncSession, + current_user: CurrentUserModel, + query_object: FileReconcileRunPageQueryModel, + is_page: bool = True, + ) -> PageModel | list[FileReconcileRunModel]: + """ + 获取文件存储对账任务列表 + """ + cls._require_admin(current_user) + run_list = await FileInfoDao.get_reconcile_run_list(query_db, query_object, is_page) + if isinstance(run_list, PageModel): + return PageModel.model_validate( + { + 'rows': [FileReconcileRunModel.model_validate(row, by_name=True) for row in run_list.rows], + 'page_num': run_list.page_num, + 'page_size': run_list.page_size, + 'total': run_list.total, + 'has_next': run_list.has_next, + }, + by_name=True, + ) + return [FileReconcileRunModel.model_validate(row, by_name=True) for row in run_list] + + @classmethod + async def get_reconcile_issue_list_services( + cls, + query_db: AsyncSession, + current_user: CurrentUserModel, + query_object: FileReconcileIssuePageQueryModel, + is_page: bool = True, + ) -> PageModel | list[FileReconcileIssueModel]: + """ + 获取文件存储对账异常列表 + """ + cls._require_admin(current_user) + issue_list = await FileInfoDao.get_reconcile_issue_list(query_db, query_object, is_page) + issue_rows = issue_list.rows if isinstance(issue_list, PageModel) else issue_list + issue_models = [] + for issue_row in issue_rows: + issue_model = FileReconcileIssueModel.model_validate(issue_row, by_name=True) + issue_model.available_actions = cls._get_available_actions(issue_model) + issue_models.append(issue_model) + if isinstance(issue_list, PageModel): + return PageModel.model_validate( + { + 'rows': issue_models, + 'page_num': issue_list.page_num, + 'page_size': issue_list.page_size, + 'total': issue_list.total, + 'has_next': issue_list.has_next, + }, + by_name=True, + ) + return issue_models + + @classmethod + async def get_reconcile_stats_services( + cls, + query_db: AsyncSession, + current_user: CurrentUserModel, + ) -> FileReconcileStatsModel: + """ + 获取文件存储对账统计 + """ + cls._require_admin(current_user) + stats = await FileInfoDao.get_reconcile_stats(query_db) + latest_run = stats.pop('latest_run') + stats['latest_run'] = ( + FileReconcileRunModel.model_validate(cls._build_run_data(latest_run), by_name=True) if latest_run else None + ) + return FileReconcileStatsModel.model_validate(stats, by_name=True) + + @classmethod + async def handle_reconcile_issue_services( + cls, + query_db: AsyncSession, + current_user: CurrentUserModel, + issue_id: int, + handle: FileReconcileHandleModel, + request: Request | None = None, + ) -> CrudResponseModel: + """ + 处理文件存储对账异常 + + :param query_db: orm对象 + :param current_user: 当前用户对象 + :param issue_id: 异常ID + :param handle: 处理参数 + :param request: Request对象 + :return: 处理结果 + """ + cls._require_admin(current_user) + if await FileInfoDao.has_running_reconcile_run(query_db): + raise ServiceException(message='文件存储对账任务运行中，请等待扫描完成后再处理异常') + issue = await FileInfoDao.get_reconcile_issue_for_update(query_db, issue_id) + if issue is None: + raise ServiceException(message='文件存储对账异常不存在') + available_actions = cls._get_available_actions(issue) + if handle.action not in available_actions: + await query_db.rollback() + raise ServiceException(message='当前异常状态不支持该处理动作') + + file_id = issue.file_id + issue_type = issue.issue_type + operation_location = { + 'actualRoot': issue.actual_root, + 'actualKey': issue.actual_key, + 'expectedRoot': issue.expected_root, + 'expectedKey': issue.expected_key, + } + performed_move: tuple[str, str, str, str] | None = None + current_time = datetime.now() + try: + file_id, performed_move = await cls._apply_reconcile_action( + query_db, + issue, + handle, + current_user, + current_time, + ) + await query_db.commit() + except ServiceException: + await query_db.rollback() + if performed_move: + await cls._rollback_file_move(performed_move) + raise + except Exception as exc: + await query_db.rollback() + if performed_move: + await cls._rollback_file_move(performed_move) + if isinstance(exc, (FileExistsError, FileNotFoundError, OSError, ValueError)): + raise ServiceException(message=f'文件存储异常处理失败：{exc}') from exc + raise + + if file_id: + await FileAuditService.enqueue_file_audit( + request, + current_user, + file_id, + 'reconcile', + 'completed', + operation_detail={ + 'issueId': issue_id, + 'issueType': issue_type, + 'action': handle.action, + 'reason': handle.reason, + **operation_location, + }, + ) + return CrudResponseModel(is_success=True, message='文件存储异常处理成功') + + @classmethod + async def _apply_reconcile_action( + cls, + query_db: AsyncSession, + issue: SysFileReconcileIssue, + handle: FileReconcileHandleModel, + current_user: CurrentUserModel, + current_time: datetime, + ) -> tuple[str | None, tuple[str, str, str, str] | None]: + """执行已通过状态校验的对账处理动作。""" + user = cls._require_admin(current_user) + performed_move: tuple[str, str, str, str] | None = None + file_id = issue.file_id + if handle.action in {'ignore', 'reopen'}: + status = 'ignored' if handle.action == 'ignore' else 'open' + cls._mark_issue_handled(issue, status, handle, user.user_name, current_time) + elif handle.action in {'restore_source', 'move_to_trash', 'move_to_expected_root'}: + performed_move = await cls._move_issue_file(issue) + cls._mark_issue_handled(issue, 'resolved', handle, user.user_name, current_time) + elif handle.action == 'quarantine_file': + performed_move = await cls._quarantine_issue_file(issue) + cls._mark_issue_handled(issue, 'quarantined', handle, user.user_name, current_time) + elif handle.action == 'restore_quarantine': + performed_move = await cls._restore_quarantine_file(issue) + cls._mark_issue_handled(issue, 'open', handle, user.user_name, current_time) + elif handle.action == 'delete_quarantine': + await cls._delete_quarantine_file(issue) + cls._mark_issue_handled(issue, 'resolved', handle, user.user_name, current_time) + elif handle.action == 'accept_current': + file_id = await cls._accept_current_file( + query_db, + issue, + user.user_name, + handle.reason, + current_time, + ) + else: + file_id = await cls._register_orphan_file( + query_db, + issue, + handle, + current_user, + current_time, + ) + return file_id, performed_move + + @classmethod + async def _move_issue_file( + cls, + issue: SysFileReconcileIssue, + ) -> tuple[str, str, str, str]: + move = cls._get_issue_move_locations(issue) + await asyncio.to_thread(FileReconcileUtil.move_regular_file, *move) + return move + + @classmethod + async def _quarantine_issue_file( + cls, + issue: SysFileReconcileIssue, + ) -> tuple[str, str, str, str]: + source_root, source_key = cls._require_location(issue.actual_root, issue.actual_key) + quarantine_key = f'{issue.issue_id}/{source_root}/{source_key}' + move = (source_root, source_key, 'quarantine', quarantine_key) + await asyncio.to_thread(FileReconcileUtil.move_regular_file, *move) + issue.quarantine_key = quarantine_key + return move + + @classmethod + async def _restore_quarantine_file( + cls, + issue: SysFileReconcileIssue, + ) -> tuple[str, str, str, str]: + if not issue.quarantine_key: + raise ServiceException(message='隔离区文件路径不存在') + target_root, target_key = cls._require_location(issue.actual_root, issue.actual_key) + move = ('quarantine', issue.quarantine_key, target_root, target_key) + await asyncio.to_thread(FileReconcileUtil.move_regular_file, *move) + issue.quarantine_key = None + return move + + @staticmethod + async def _delete_quarantine_file(issue: SysFileReconcileIssue) -> None: + if not issue.quarantine_key: + raise ServiceException(message='隔离区文件路径不存在') + await asyncio.to_thread(FileReconcileUtil.delete_quarantine_file, issue.quarantine_key) + issue.quarantine_key = None + + @classmethod + async def _accept_current_file( + cls, + query_db: AsyncSession, + issue: SysFileReconcileIssue, + handled_by: str, + reason: str, + current_time: datetime, + ) -> str: + if not issue.file_id: + raise ServiceException(message='异常未关联文件信息') + file_info = await FileInfoDao.get_file_info_for_reconcile(query_db, issue.file_id) + if file_info is None: + raise ServiceException(message='异常关联的文件信息不存在') + root_name, relative_key = cls._require_location( + issue.actual_root or issue.expected_root, + issue.actual_key or issue.expected_key, + ) + file_path = FileReconcileUtil.resolve_location(root_name, relative_key) + file_size, file_hash = await asyncio.to_thread( + FileReconcileUtil.calculate_file_integrity, + file_path, + ) + file_info.file_size = file_size + file_info.file_hash = file_hash + file_info.update_by = handled_by + file_info.update_time = current_time + await FileInfoDao.resolve_file_integrity_issues( + query_db, + issue.file_id, + current_time, + handled_by, + reason, + ) + return issue.file_id + + @classmethod + async def _register_orphan_file( + cls, + query_db: AsyncSession, + issue: SysFileReconcileIssue, + handle: FileReconcileHandleModel, + current_user: CurrentUserModel, + current_time: datetime, + ) -> str: + user = cls._require_admin(current_user) + access_type, storage_key = cls._require_location(issue.actual_root, issue.actual_key) + if issue.issue_type != 'orphan_file' or access_type not in {'public', 'private'}: + raise ServiceException(message='仅公开或受保护存储区的孤立文件可以登记') + if await FileInfoDao.get_file_info_by_storage_key(query_db, storage_key, access_type) is not None: + raise ServiceException(message='该物理文件已登记到文件信息表') + stored_name = storage_key.rsplit('/', 1)[-1] + extension = UploadUtil.get_file_extension(stored_name) + if extension not in UploadConfig.DEFAULT_ALLOWED_EXTENSION: + raise ServiceException(message='孤立文件扩展名不在允许范围内') + original_name = UploadUtil.get_original_filename(handle.original_name or stored_name) + if not original_name or UploadUtil.get_file_extension(original_name) != extension: + raise ServiceException(message='原始文件名扩展名必须与物理文件一致') + file_path = FileReconcileUtil.resolve_location(access_type, storage_key) + file_size, file_hash = await asyncio.to_thread( + FileReconcileUtil.calculate_file_integrity, + file_path, + ) + file_id = str(uuid.uuid4()) + await FileInfoDao.add_file_info_dao( + query_db, + FileInfoModel( + fileId=file_id, + originalName=original_name, + storedName=stored_name, + storageKey=storage_key, + storageType='local', + accessType=access_type, + uploadUserId=user.user_id, + ownerUserId=user.user_id, + deptId=user.dept_id, + extension=extension, + contentType=mimetypes.guess_type(original_name)[0] or 'application/octet-stream', + fileSize=file_size, + fileHash=file_hash, + status='active', + createBy=user.user_name, + createTime=current_time, + updateBy=user.user_name, + updateTime=current_time, + delFlag='0', + ), + ) + issue.file_id = file_id + cls._mark_issue_handled(issue, 'resolved', handle, user.user_name, current_time) + return file_id + + @classmethod + def _get_available_actions( + cls, + issue: SysFileReconcileIssue | FileReconcileIssueModel, + ) -> list[FileReconcileAction]: + if issue.quarantine_key or issue.status == 'quarantined': + return ['restore_quarantine', 'delete_quarantine'] + if issue.status == 'ignored': + return ['reopen'] + if issue.status != 'open': + return [] + actions: list[FileReconcileAction] = ['ignore'] + actions.extend(cls.ISSUE_ACTIONS.get(issue.issue_type, [])) + if issue.issue_type == 'orphan_file' and issue.actual_root not in {'public', 'private'}: + actions = [action for action in actions if action != 'register_orphan'] + return actions + + @classmethod + def _get_issue_move_locations( + cls, + issue: SysFileReconcileIssue, + ) -> tuple[str, str, str, str]: + source_root, source_key = cls._require_location(issue.actual_root, issue.actual_key) + target_root, target_key = cls._require_location(issue.expected_root, issue.expected_key) + return source_root, source_key, target_root, target_key + + @staticmethod + def _require_location(root_name: str | None, relative_key: str | None) -> tuple[str, str]: + if not root_name or not relative_key: + raise ServiceException(message='异常记录缺少可处理的存储位置') + if root_name not in {'public', 'private', 'trash', 'quarantine'}: + raise ServiceException(message='异常记录的存储区域不合法') + return root_name, relative_key + + @staticmethod + def _mark_issue_handled( + issue: SysFileReconcileIssue, + status: str, + handle: FileReconcileHandleModel, + handled_by: str, + handled_time: datetime, + ) -> None: + issue.status = status + issue.handle_action = handle.action + issue.handle_reason = handle.reason + issue.handled_by = handled_by + issue.handled_time = handled_time + + @classmethod + async def _rollback_file_move(cls, move: tuple[str, str, str, str]) -> None: + source_root, source_key, target_root, target_key = move + try: + await asyncio.to_thread( + FileReconcileUtil.move_regular_file, + target_root, + target_key, + source_root, + source_key, + ) + except Exception: + logger.exception('文件存储异常处理数据库回滚后，物理文件补偿失败') + + @staticmethod + def _build_run_data(reconcile_run: SysFileReconcileRun) -> dict[str, Any]: + return { + 'run_id': reconcile_run.run_id, + 'trigger_type': reconcile_run.trigger_type, + 'status': reconcile_run.status, + 'check_hash': reconcile_run.check_hash == '1', + 'scanned_file_count': reconcile_run.scanned_file_count or 0, + 'scanned_storage_count': reconcile_run.scanned_storage_count or 0, + 'issue_count': reconcile_run.issue_count or 0, + 'new_issue_count': reconcile_run.new_issue_count or 0, + 'resolved_issue_count': reconcile_run.resolved_issue_count or 0, + 'started_by': reconcile_run.started_by, + 'started_time': reconcile_run.started_time, + 'finished_time': reconcile_run.finished_time, + 'error_message': reconcile_run.error_message, + } + + @staticmethod + def _require_admin(current_user: CurrentUserModel | None) -> Any: + user = current_user.user if current_user else None + if user is None or not user.admin or not user.user_name: + raise ServiceException(message='仅系统管理员可以使用文件存储对账功能') + return user diff --git a/ruoyi-fastapi-backend/module_admin/service/log_service.py b/ruoyi-fastapi-backend/module_admin/service/log_service.py index 3560f39..fdb334b 100644 --- a/ruoyi-fastapi-backend/module_admin/service/log_service.py +++ b/ruoyi-fastapi-backend/module_admin/service/log_service.py @@ -14,7 +14,9 @@ from config.database import AsyncSessionLocal from config.env import LogConfig from exceptions.exception import ServiceException from middlewares.trace_middleware.ctx import TraceCtx +from module_admin.dao.file_access_dao import FileAccessLogDao from module_admin.dao.log_dao import LoginLogDao, OperationLogDao +from module_admin.entity.vo.file_vo import FileAccessLogModel from module_admin.entity.vo.log_vo import ( DeleteLoginLogModel, DeleteOperLogModel, @@ -348,6 +350,19 @@ class LogQueueService: payload = LogSanitizer.sanitize_data(operation_log.model_dump(by_alias=True, exclude_none=True)) await cls._xadd_event(request.app.state.redis, 'operation', payload, source) + @classmethod + async def enqueue_file_access_log(cls, request: Request, file_access_log: FileAccessLogModel, source: str) -> None: + """ + 文件访问审计日志入队 + + :param request: Request对象 + :param file_access_log: 文件访问审计日志模型 + :param source: 日志来源 + :return: None + """ + payload = LogSanitizer.sanitize_data(file_access_log.model_dump(by_alias=True, exclude_none=True)) + await cls._xadd_event(request.app.state.redis, 'file_access', payload, source) + class LogAggregatorService: """ @@ -484,7 +499,7 @@ class LogAggregatorService: event_type = data.get('event_type') event_id = data.get('event_id') payload_raw = data.get('payload') or '{}' - if event_type not in {'login', 'operation'}: + if event_type not in {'login', 'operation', 'file_access'}: ack_ids.append(message_id) continue acquired = await cls._acquire_dedup(redis, event_id) @@ -497,6 +512,8 @@ class LogAggregatorService: await LoginLogDao.add_login_log_dao(session, LogininforModel(**payload)) elif event_type == 'operation': await OperationLogDao.add_operation_log_dao(session, OperLogModel(**payload)) + elif event_type == 'file_access': + await FileAccessLogDao.add_file_access_log_dao(session, FileAccessLogModel(**payload)) ack_ids.append(message_id) if ack_ids: await session.commit() diff --git a/ruoyi-fastapi-backend/module_task/__init__.py b/ruoyi-fastapi-backend/module_task/__init__.py index 1f4b412..8cd2bdf 100644 --- a/ruoyi-fastapi-backend/module_task/__init__.py +++ b/ruoyi-fastapi-backend/module_task/__init__.py @@ -1 +1 @@ -from . import scheduler_test # noqa: F401 +from . import file_task, scheduler_test # noqa: F401 diff --git a/ruoyi-fastapi-backend/module_task/file_task.py b/ruoyi-fastapi-backend/module_task/file_task.py new file mode 100644 index 0000000..99a217a --- /dev/null +++ b/ruoyi-fastapi-backend/module_task/file_task.py @@ -0,0 +1,66 @@ +from config.database import AsyncSessionLocal +from module_admin.service.file_business_service import FileRetentionNoticeService +from module_admin.service.file_service import FileLifecycleService, FileReconcileService +from utils.log_util import logger + +MAX_TASK_BATCHES = 100 + + +async def scan_retention_reminders(remind_days: int = 7, batch_size: int = 500) -> None: + """ + 扫描文件保留期限并生成提醒 + + :param remind_days: 提前提醒天数 + :param batch_size: 单类提醒单批处理数量 + :return: None + """ + expiring_count = 0 + expired_count = 0 + async with AsyncSessionLocal() as query_db: + for _ in range(MAX_TASK_BATCHES): + scan_result = await FileRetentionNoticeService.scan_file_retention_notices_services( + query_db, + remind_days=remind_days, + batch_size=batch_size, + ) + expiring_count += scan_result.expiring_count + expired_count += scan_result.expired_count + if scan_result.expiring_count < batch_size and scan_result.expired_count < batch_size: + break + else: + logger.warning('文件保留期限提醒扫描达到最大批次数，请检查待处理文件数量') + logger.info(f'文件保留期限提醒扫描完成，即将到期{expiring_count}个，已到期{expired_count}个') + + +async def purge_recycle_bin(retention_days: int = 30, batch_size: int = 100) -> None: + """ + 永久清理超过保留期限的回收站文件 + + :param retention_days: 回收站保留天数 + :param batch_size: 单批处理数量 + :return: None + """ + purge_count = 0 + async with AsyncSessionLocal() as query_db: + for _ in range(MAX_TASK_BATCHES): + current_count = await FileLifecycleService.purge_recycle_bin_services( + query_db, + retention_days=retention_days, + batch_size=batch_size, + ) + purge_count += current_count + if current_count < batch_size: + break + else: + logger.warning('回收站永久清理达到最大批次数，请检查待处理文件数量') + logger.info(f'回收站永久清理完成，共清理{purge_count}个文件') + + +async def reconcile_file_storage(check_hash: bool = False) -> None: + """ + 执行数据库和本地文件系统双向对账 + + :param check_hash: 是否校验文件SHA-256 + :return: None + """ + await FileReconcileService.run_scheduled_reconcile_services(check_hash) diff --git a/ruoyi-fastapi-backend/scripts/migrate_legacy_files.py b/ruoyi-fastapi-backend/scripts/migrate_legacy_files.py new file mode 100644 index 0000000..20dfa05 --- /dev/null +++ b/ruoyi-fastapi-backend/scripts/migrate_legacy_files.py @@ -0,0 +1,289 @@ +import argparse +import asyncio +import hashlib +import mimetypes +import stat +import uuid +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path + +import aiofiles +from pydantic import ValidationError + +from config.database import AsyncSessionLocal +from config.env import UploadConfig +from module_admin.dao.file_info_dao import FileInfoDao +from module_admin.entity.vo.file_vo import FileInfoModel +from utils.log_util import logger +from utils.upload_util import UploadUtil + +FILE_NAME_MAX_LENGTH = 255 +STORAGE_KEY_MAX_LENGTH = 500 +EXTENSION_MAX_LENGTH = 20 +CONTENT_TYPE_MAX_LENGTH = 255 + + +@dataclass(frozen=True) +class FileSignature: + """ + 文件稳定性签名 + """ + + file_size: int + modified_time_ns: int + changed_time_ns: int + device_id: int + inode: int + + +@dataclass(frozen=True) +class LegacyFileInfo: + """ + 历史文件信息 + """ + + filepath: Path + storage_key: str + filename: str + extension: str + content_type: str | None + file_size: int + file_time: datetime + signature: FileSignature + + +def get_file_signature(filepath: Path) -> FileSignature: + """ + 获取普通文件的稳定性签名 + + :param filepath: 文件路径 + :return: 文件稳定性签名 + """ + file_stat = filepath.lstat() + if not stat.S_ISREG(file_stat.st_mode): + raise ValueError('文件不是普通文件') + return FileSignature( + file_size=file_stat.st_size, + modified_time_ns=file_stat.st_mtime_ns, + changed_time_ns=file_stat.st_ctime_ns, + device_id=file_stat.st_dev, + inode=file_stat.st_ino, + ) + + +def get_legacy_file_skip_reason( + storage_key: str, + filename: str, + extension: str, + content_type: str | None, + file_size: int, +) -> str | None: + """ + 获取历史文件跳过原因 + + :param storage_key: 存储相对路径 + :param filename: 存储文件名 + :param extension: 文件扩展名 + :param content_type: 文件内容类型 + :param file_size: 文件大小 + :return: 跳过原因 + """ + if extension not in UploadConfig.DEFAULT_ALLOWED_EXTENSION: + return '文件扩展名不在允许列表' + if file_size > UploadConfig.MAX_FILE_SIZE: + return f'文件大小超过{UploadConfig.MAX_FILE_SIZE // 1024 // 1024}MB' + if len(filename) > FILE_NAME_MAX_LENGTH: + return f'文件名超过{FILE_NAME_MAX_LENGTH}个字符' + if len(storage_key) > STORAGE_KEY_MAX_LENGTH: + return f'存储相对路径超过{STORAGE_KEY_MAX_LENGTH}个字符' + if len(extension) > EXTENSION_MAX_LENGTH: + return f'文件扩展名超过{EXTENSION_MAX_LENGTH}个字符' + if content_type and len(content_type) > CONTENT_TYPE_MAX_LENGTH: + return f'文件内容类型超过{CONTENT_TYPE_MAX_LENGTH}个字符' + return None + + +def collect_legacy_files() -> tuple[list[LegacyFileInfo], int]: + """ + 收集待迁移的历史文件 + + :return: 符合要求的文件信息和跳过数量 + """ + upload_root = Path(UploadConfig.UPLOAD_PATH).resolve() + legacy_files = [] + skipped_count = 0 + for filepath in upload_root.rglob('*'): + try: + signature = get_file_signature(filepath) + except ValueError: + continue + except OSError as exc: + logger.warning(f'跳过无法读取的历史文件路径: {filepath}，原因: {exc.__class__.__name__}') + skipped_count += 1 + continue + + storage_key = filepath.relative_to(upload_root).as_posix() + filename = filepath.name + extension = UploadUtil.get_file_extension(filename) + content_type = mimetypes.guess_type(filename)[0] + skip_reason = get_legacy_file_skip_reason( + storage_key, + filename, + extension, + content_type, + signature.file_size, + ) + if skip_reason: + logger.warning(f'跳过历史文件: {storage_key}，原因: {skip_reason}') + skipped_count += 1 + continue + + legacy_files.append( + LegacyFileInfo( + filepath=filepath, + storage_key=storage_key, + filename=filename, + extension=extension, + content_type=content_type, + file_size=signature.file_size, + file_time=datetime.fromtimestamp(signature.modified_time_ns / 1_000_000_000), + signature=signature, + ) + ) + return legacy_files, skipped_count + + +async def calculate_file_hash(filepath: Path, expected_signature: FileSignature | None = None) -> str: + """ + 计算稳定文件的SHA-256 + + :param filepath: 文件路径 + :param expected_signature: 扫描阶段记录的文件签名 + :return: 文件SHA-256 + """ + before_signature = await asyncio.to_thread(get_file_signature, filepath) + if expected_signature is not None and before_signature != expected_signature: + raise ValueError('文件在扫描后发生变化') + + file_hasher = hashlib.sha256() + async with aiofiles.open(filepath, 'rb') as source_file: + while chunk := await source_file.read(1024 * 1024): + file_hasher.update(chunk) + + after_signature = await asyncio.to_thread(get_file_signature, filepath) + if before_signature != after_signature: + raise ValueError('文件在哈希计算期间发生变化') + return file_hasher.hexdigest() + + +async def build_legacy_file_info(legacy_file: LegacyFileInfo) -> FileInfoModel: + """ + 构造历史文件信息模型 + + :param legacy_file: 历史文件信息 + :return: 文件信息模型 + """ + return FileInfoModel( + fileId=str(uuid.uuid4()), + originalName=legacy_file.filename, + storedName=legacy_file.filename, + storageKey=legacy_file.storage_key, + accessType='public', + extension=legacy_file.extension, + contentType=legacy_file.content_type, + fileSize=legacy_file.file_size, + fileHash=await calculate_file_hash(legacy_file.filepath, legacy_file.signature), + createBy='migration', + createTime=legacy_file.file_time, + updateBy='migration', + updateTime=legacy_file.file_time, + ) + + +async def migrate_legacy_files( + dry_run: bool = False, + batch_size: int = 100, + maintenance_confirmed: bool = False, +) -> tuple[int, int]: + """ + 将公开目录中的历史文件登记到文件信息表 + + :param dry_run: 是否仅扫描不写入数据库 + :param batch_size: 每批提交数量 + :param maintenance_confirmed: 是否已确认停止公开文件上传 + :return: 新增数量和跳过数量 + """ + if batch_size < 1: + raise ValueError('每批提交数量必须大于0') + if not dry_run and not maintenance_confirmed: + raise ValueError('正式迁移前必须停止公开文件上传并确认维护窗口') + + legacy_files, skipped_count = await asyncio.to_thread(collect_legacy_files) + added_count = 0 + pending_count = 0 + async with AsyncSessionLocal() as session: + for legacy_file in legacy_files: + if await FileInfoDao.get_file_info_by_storage_key(session, legacy_file.storage_key): + skipped_count += 1 + continue + + try: + file_info = await build_legacy_file_info(legacy_file) + except (OSError, ValidationError, ValueError) as exc: + logger.warning(f'跳过无法稳定迁移的历史文件: {legacy_file.storage_key}，原因: {exc.__class__.__name__}') + skipped_count += 1 + continue + + if await FileInfoDao.get_file_info_by_storage_key(session, legacy_file.storage_key): + skipped_count += 1 + continue + if dry_run: + logger.info(f'待登记历史文件: {legacy_file.storage_key}') + added_count += 1 + continue + + await FileInfoDao.add_file_info_dao(session, file_info) + pending_count += 1 + if pending_count == batch_size: + await session.commit() + added_count += pending_count + pending_count = 0 + + if not dry_run and pending_count: + await session.commit() + added_count += pending_count + return added_count, skipped_count + + +def parse_args() -> argparse.Namespace: + """ + 解析命令行参数 + + :return: 命令行参数 + """ + parser = argparse.ArgumentParser(description='登记公开目录中的历史文件') + parser.add_argument('--env', type=str, default='', help='运行环境') + parser.add_argument('--dry-run', action='store_true', help='执行完整预检但不写入数据库') + parser.add_argument('--batch-size', type=int, default=100, help='每批提交数量') + parser.add_argument( + '--confirm-maintenance', + action='store_true', + help='确认正式迁移期间已经停止公开文件上传', + ) + args = parser.parse_args() + if not args.dry_run and not args.confirm_maintenance: + parser.error('正式迁移必须指定--confirm-maintenance并停止公开文件上传') + return args + + +if __name__ == '__main__': + args = parse_args() + added, skipped = asyncio.run( + migrate_legacy_files( + dry_run=args.dry_run, + batch_size=args.batch_size, + maintenance_confirmed=args.confirm_maintenance, + ) + ) + logger.info(f'历史文件登记完成，新增{added}个，跳过{skipped}个') diff --git a/ruoyi-fastapi-backend/sql/ruoyi-fastapi-pg.sql b/ruoyi-fastapi-backend/sql/ruoyi-fastapi-pg.sql index 0e68fe9..05f5581 100644 --- a/ruoyi-fastapi-backend/sql/ruoyi-fastapi-pg.sql +++ b/ruoyi-fastapi-backend/sql/ruoyi-fastapi-pg.sql @@ -257,6 +257,7 @@ insert into sys_menu values(105, '字典管理', 1, '6', 'dict', insert into sys_menu values(106, '参数设置', 1, '7', 'config', 'system/config/index', '', '', 1, 0, 'C', '0', '0', 'system:config:list', 'edit', 'admin', current_timestamp, '', null, '参数设置菜单'); insert into sys_menu values(107, '通知公告', 1, '8', 'notice', 'system/notice/index', '', '', 1, 0, 'C', '0', '0', 'system:notice:list', 'message', 'admin', current_timestamp, '', null, '通知公告菜单'); insert into sys_menu values(108, '日志管理', 1, '9', 'log', '', '', '', 1, 0, 'M', '0', '0', '', 'log', 'admin', current_timestamp, '', null, '日志管理菜单'); +insert into sys_menu values(121, '文件管理', 1, '10', 'file', 'system/file/index', '', '', 1, 0, 'C', '0', '0', 'system:file:list', 'documentation', 'admin', current_timestamp, '', null, '文件管理菜单'); insert into sys_menu values(109, '在线用户', 2, '1', 'online', 'monitor/online/index', '', '', 1, 0, 'C', '0', '0', 'monitor:online:list', 'online', 'admin', current_timestamp, '', null, '在线用户菜单'); insert into sys_menu values(110, '定时任务', 2, '2', 'job', 'monitor/job/index', '', '', 1, 0, 'C', '0', '0', 'monitor:job:list', 'job', 'admin', current_timestamp, '', null, '定时任务菜单'); insert into sys_menu values(111, '数据监控', 2, '3', 'druid', 'monitor/druid/index', '', '', 1, 0, 'C', '0', '0', 'monitor:druid:list', 'druid', 'admin', current_timestamp, '', null, '数据监控菜单'); @@ -319,6 +320,15 @@ insert into sys_menu values(1035, '公告查询', 107, '1', '#', '', '', '', 1, insert into sys_menu values(1036, '公告新增', 107, '2', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:notice:add', '#', 'admin', current_timestamp, '', null, ''); insert into sys_menu values(1037, '公告修改', 107, '3', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:notice:edit', '#', 'admin', current_timestamp, '', null, ''); insert into sys_menu values(1038, '公告删除', 107, '4', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:notice:remove', '#', 'admin', current_timestamp, '', null, ''); +-- 文件管理按钮 +insert into sys_menu values(1065, '文件查询', 121, '1', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:file:query', '#', 'admin', current_timestamp, '', null, ''); +insert into sys_menu values(1066, '文件下载', 121, '2', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:file:download', '#', 'admin', current_timestamp, '', null, ''); +insert into sys_menu values(1067, '文件删除', 121, '3', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:file:remove', '#', 'admin', current_timestamp, '', null, ''); +insert into sys_menu values(1068, '文件授权', 121, '4', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:file:edit', '#', 'admin', current_timestamp, '', null, ''); +insert into sys_menu values(1069, '文件转移', 121, '5', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:file:transfer', '#', 'admin', current_timestamp, '', null, ''); +insert into sys_menu values(1070, '文件恢复', 121, '6', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:file:restore', '#', 'admin', current_timestamp, '', null, ''); +insert into sys_menu values(1071, '文件清理', 121, '7', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:file:purge', '#', 'admin', current_timestamp, '', null, ''); +insert into sys_menu values(1072, '存储对账', 121, '8', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:file:reconcile', '#', 'admin', current_timestamp, '', null, ''); -- 操作日志按钮 insert into sys_menu values(1039, '操作查询', 500, '1', '#', '', '', '', 1, 0, 'F', '0', '0', 'monitor:operlog:query', '#', 'admin', current_timestamp, '', null, ''); insert into sys_menu values(1040, '操作删除', 500, '2', '#', '', '', '', 1, 0, 'F', '0', '0', 'monitor:operlog:remove', '#', 'admin', current_timestamp, '', null, ''); @@ -835,6 +845,9 @@ comment on table sys_job is '定时任务调度表'; insert into sys_job values(1, '系统默认（无参）', 'default', 'default', 'module_task.scheduler_test.job', null, null, '0/10 * * * * ?', '3', '1', '1', 'admin', current_timestamp, '', null, ''); insert into sys_job values(2, '系统默认（有参）', 'default', 'default', 'module_task.scheduler_test.job', 'test', null, '0/15 * * * * ?', '3', '1', '1', 'admin', current_timestamp, '', null, ''); insert into sys_job values(3, '系统默认（多参）', 'default', 'default', 'module_task.scheduler_test.job', 'new', '{test: 111}', '0/20 * * * * ?', '3', '1', '1', 'admin', current_timestamp, '', null, ''); +insert into sys_job values(4, '文件保留期限提醒', 'default', 'default', 'module_task.file_task.scan_retention_reminders', null, '{"remind_days": 7, "batch_size": 500}', '0 0 1 * * ?', '3', '1', '0', 'admin', current_timestamp, '', null, '每天扫描即将到期和已到期的受保护文件'); +insert into sys_job values(5, '回收站永久清理', 'default', 'default', 'module_task.file_task.purge_recycle_bin', null, '{"retention_days": 30, "batch_size": 100}', '0 0 2 * * ?', '3', '1', '1', 'admin', current_timestamp, '', null, '永久清理超过保留期限的回收站文件，默认暂停'); +insert into sys_job values(6, '文件存储对账', 'default', 'default', 'module_task.file_task.reconcile_file_storage', null, '{"check_hash": false}', '0 0 3 * * ?', '3', '1', '1', 'admin', current_timestamp, '', null, '校验文件信息表和本地存储一致性，默认暂停'); -- ---------------------------- -- 16、定时任务调度日志表 @@ -1091,6 +1104,329 @@ comment on column ai_chat_config.image_max_size_mb is '图片最大大小(MB)'; comment on column ai_chat_config.create_time is '创建时间'; comment on column ai_chat_config.update_time is '更新时间'; +-- ---------------------------- +-- 22、文件信息表 +-- ---------------------------- +drop table if exists sys_file_info; +create table sys_file_info ( + file_id varchar(36) not null, + original_name varchar(255) not null, + stored_name varchar(255) not null, + storage_key varchar(500) not null, + storage_type varchar(20) not null default 'local', + access_type varchar(20) not null default 'public', + upload_user_id bigint, + uploader_access_enabled char(1) not null default '1', + owner_user_id bigint, + dept_id bigint, + acl_version integer not null default 0, + business_type varchar(50), + business_id varchar(64), + extension varchar(20) not null default '', + content_type varchar(255), + file_size bigint not null default 0, + file_hash varchar(64) not null, + status varchar(20) not null default 'active', + create_by varchar(64) default '', + create_time timestamp(0) not null, + update_by varchar(64) default '', + update_time timestamp(0) not null, + expire_time timestamp(0), + deleted_time timestamp(0), + del_flag char(1) not null default '0', + primary key (file_id) +); +create index idx_sys_file_info_access_status on sys_file_info(access_type, status); +create index idx_sys_file_info_owner_status on sys_file_info(owner_user_id, status); +create index idx_sys_file_info_dept_status on sys_file_info(dept_id, status); +create index idx_sys_file_info_status_deleted_time on sys_file_info(status, deleted_time); +create unique index uk_sys_file_info_storage_location on sys_file_info(storage_type, access_type, storage_key); +comment on table sys_file_info is '文件信息表'; +comment on column sys_file_info.file_id is '文件ID'; +comment on column sys_file_info.original_name is '原始文件名'; +comment on column sys_file_info.stored_name is '存储文件名'; +comment on column sys_file_info.storage_key is '存储相对路径'; +comment on column sys_file_info.storage_type is '存储类型'; +comment on column sys_file_info.access_type is '访问类型'; +comment on column sys_file_info.upload_user_id is '上传用户ID'; +comment on column sys_file_info.uploader_access_enabled is '是否保留上传人访问权限'; +comment on column sys_file_info.owner_user_id is '所有者用户ID'; +comment on column sys_file_info.dept_id is '所属部门ID'; +comment on column sys_file_info.acl_version is '访问控制版本'; +comment on column sys_file_info.business_type is '业务类型'; +comment on column sys_file_info.business_id is '业务ID'; +comment on column sys_file_info.extension is '文件扩展名'; +comment on column sys_file_info.content_type is '内容类型'; +comment on column sys_file_info.file_size is '文件大小'; +comment on column sys_file_info.file_hash is '文件SHA-256'; +comment on column sys_file_info.status is '文件状态'; +comment on column sys_file_info.create_by is '创建者'; +comment on column sys_file_info.create_time is '创建时间'; +comment on column sys_file_info.update_by is '更新者'; +comment on column sys_file_info.update_time is '更新时间'; +comment on column sys_file_info.expire_time is '过期时间'; +comment on column sys_file_info.deleted_time is '移入回收站时间'; +comment on column sys_file_info.del_flag is '删除标志'; + +-- ---------------------------- +-- 23、文件业务引用表 +-- ---------------------------- +drop table if exists sys_file_reference; +create table sys_file_reference ( + reference_id bigserial not null, + file_id varchar(36) not null, + business_type varchar(50) not null, + business_id varchar(64) not null, + business_name varchar(255), + retention_expire_time timestamp(0), + create_by varchar(64) default '', + create_time timestamp(0) not null, + primary key (reference_id) +); +create unique index uk_sys_file_reference_business on sys_file_reference(file_id, business_type, business_id); +create index idx_sys_file_reference_file on sys_file_reference(file_id); +create index idx_sys_file_reference_business on sys_file_reference(business_type, business_id); +comment on table sys_file_reference is '文件业务引用表'; +comment on column sys_file_reference.reference_id is '引用ID'; +comment on column sys_file_reference.file_id is '文件ID'; +comment on column sys_file_reference.business_type is '业务类型'; +comment on column sys_file_reference.business_id is '业务ID'; +comment on column sys_file_reference.business_name is '业务名称'; +comment on column sys_file_reference.retention_expire_time is '保留期限到期时间'; +comment on column sys_file_reference.create_by is '创建者'; +comment on column sys_file_reference.create_time is '创建时间'; + +-- ---------------------------- +-- 24、文件业务保留策略表 +-- ---------------------------- +drop table if exists sys_file_retention_policy; +create table sys_file_retention_policy ( + business_type varchar(50) not null, + retention_days integer not null, + status char(1) not null default '0', + remark varchar(500), + create_by varchar(64) default '', + create_time timestamp(0) not null, + update_by varchar(64) default '', + update_time timestamp(0) not null, + primary key (business_type) +); +comment on table sys_file_retention_policy is '文件业务保留策略表'; +comment on column sys_file_retention_policy.business_type is '业务类型'; +comment on column sys_file_retention_policy.retention_days is '保留天数'; +comment on column sys_file_retention_policy.status is '状态（0启用 1停用）'; +comment on column sys_file_retention_policy.remark is '备注'; +comment on column sys_file_retention_policy.create_by is '创建者'; +comment on column sys_file_retention_policy.create_time is '创建时间'; +comment on column sys_file_retention_policy.update_by is '更新者'; +comment on column sys_file_retention_policy.update_time is '更新时间'; + +-- ---------------------------- +-- 25、文件保留期限提醒表 +-- ---------------------------- +drop table if exists sys_file_retention_notice; +create table sys_file_retention_notice ( + notice_id bigserial not null, + file_id varchar(36) not null, + notice_type varchar(20) not null, + expire_time timestamp(0) not null, + status char(1) not null default '0', + create_time timestamp(0) not null, + read_by varchar(64) default '', + read_time timestamp(0), + primary key (notice_id) +); +create unique index uk_sys_file_retention_notice_file_type_time + on sys_file_retention_notice(file_id, notice_type, expire_time); +create index idx_sys_file_retention_notice_file on sys_file_retention_notice(file_id); +create index idx_sys_file_retention_notice_status_time on sys_file_retention_notice(status, create_time); +comment on table sys_file_retention_notice is '文件保留期限提醒表'; +comment on column sys_file_retention_notice.notice_id is '提醒ID'; +comment on column sys_file_retention_notice.file_id is '文件ID'; +comment on column sys_file_retention_notice.notice_type is '提醒类型'; +comment on column sys_file_retention_notice.expire_time is '文件过期时间'; +comment on column sys_file_retention_notice.status is '状态（0未读 1已读 2已失效）'; +comment on column sys_file_retention_notice.create_time is '创建时间'; +comment on column sys_file_retention_notice.read_by is '读取者'; +comment on column sys_file_retention_notice.read_time is '读取时间'; + +-- ---------------------------- +-- 26、文件访问控制表 +-- ---------------------------- +drop table if exists sys_file_acl; +create table sys_file_acl ( + acl_id bigserial not null, + file_id varchar(36) not null, + subject_type varchar(20) not null, + subject_id bigint not null, + permission varchar(20) not null default 'download', + effect varchar(10) not null default 'allow', + include_children char(1) not null default '0', + expire_time timestamp(0), + create_by varchar(64) default '', + create_time timestamp(0) not null, + del_flag char(1) not null default '0', + primary key (acl_id) +); +create unique index uk_sys_file_acl_subject_permission on sys_file_acl(file_id, subject_type, subject_id, permission); +create index idx_sys_file_acl_file_status on sys_file_acl(file_id, del_flag, expire_time); +create index idx_sys_file_acl_subject on sys_file_acl(subject_type, subject_id); +comment on table sys_file_acl is '文件访问控制表'; +comment on column sys_file_acl.acl_id is '访问控制ID'; +comment on column sys_file_acl.file_id is '文件ID'; +comment on column sys_file_acl.subject_type is '主体类型'; +comment on column sys_file_acl.subject_id is '主体ID'; +comment on column sys_file_acl.permission is '权限类型'; +comment on column sys_file_acl.effect is '授权效果'; +comment on column sys_file_acl.include_children is '部门是否包含下级'; +comment on column sys_file_acl.expire_time is '授权过期时间'; +comment on column sys_file_acl.create_by is '创建者'; +comment on column sys_file_acl.create_time is '创建时间'; +comment on column sys_file_acl.del_flag is '删除标志'; + +-- ---------------------------- +-- 27、文件访问审计表 +-- ---------------------------- +drop table if exists sys_file_access_log; +create table sys_file_access_log ( + audit_id bigserial not null, + file_id varchar(36) not null, + action varchar(20) not null, + actor_user_id bigint, + actor_name varchar(64) default '', + result varchar(20) not null, + request_id varchar(64) default '', + trace_id varchar(64) default '', + ip_address varchar(128) default '', + user_agent varchar(500) default '', + bytes_sent bigint not null default 0, + error_message varchar(500) default '', + operation_detail text, + access_time timestamp(0) not null, + primary key (audit_id) +); +create index idx_sys_file_access_log_file_time on sys_file_access_log(file_id, access_time); +create index idx_sys_file_access_log_actor_time on sys_file_access_log(actor_user_id, access_time); +comment on table sys_file_access_log is '文件访问审计表'; +comment on column sys_file_access_log.audit_id is '审计ID'; +comment on column sys_file_access_log.file_id is '文件ID'; +comment on column sys_file_access_log.action is '操作类型'; +comment on column sys_file_access_log.actor_user_id is '操作用户ID'; +comment on column sys_file_access_log.actor_name is '操作用户名称'; +comment on column sys_file_access_log.result is '操作结果'; +comment on column sys_file_access_log.request_id is '请求ID'; +comment on column sys_file_access_log.trace_id is '链路ID'; +comment on column sys_file_access_log.ip_address is '客户端地址'; +comment on column sys_file_access_log.user_agent is '用户代理'; +comment on column sys_file_access_log.bytes_sent is '发送字节数'; +comment on column sys_file_access_log.error_message is '失败原因'; +comment on column sys_file_access_log.operation_detail is '操作详情'; +comment on column sys_file_access_log.access_time is '访问时间'; + +-- ---------------------------- +-- 28、文件存储对账任务表 +-- ---------------------------- +drop table if exists sys_file_reconcile_run; +create table sys_file_reconcile_run ( + run_id varchar(36) not null, + trigger_type varchar(20) not null, + status varchar(20) not null, + check_hash char(1) not null default '0', + lock_name varchar(32), + scanned_file_count bigint not null default 0, + scanned_storage_count bigint not null default 0, + issue_count bigint not null default 0, + new_issue_count bigint not null default 0, + resolved_issue_count bigint not null default 0, + started_by varchar(64) default '', + started_time timestamp(0) not null, + finished_time timestamp(0), + error_message text, + primary key (run_id) +); +create unique index uk_sys_file_reconcile_run_lock on sys_file_reconcile_run(lock_name); +create index idx_sys_file_reconcile_run_status_time on sys_file_reconcile_run(status, started_time); +comment on table sys_file_reconcile_run is '文件存储对账任务表'; +comment on column sys_file_reconcile_run.run_id is '任务ID'; +comment on column sys_file_reconcile_run.trigger_type is '触发类型'; +comment on column sys_file_reconcile_run.status is '任务状态'; +comment on column sys_file_reconcile_run.check_hash is '是否校验文件摘要'; +comment on column sys_file_reconcile_run.lock_name is '运行锁名称'; +comment on column sys_file_reconcile_run.scanned_file_count is '扫描文件记录数'; +comment on column sys_file_reconcile_run.scanned_storage_count is '扫描物理文件数'; +comment on column sys_file_reconcile_run.issue_count is '发现异常数'; +comment on column sys_file_reconcile_run.new_issue_count is '新增或重新出现异常数'; +comment on column sys_file_reconcile_run.resolved_issue_count is '自动恢复异常数'; +comment on column sys_file_reconcile_run.started_by is '发起人'; +comment on column sys_file_reconcile_run.started_time is '开始时间'; +comment on column sys_file_reconcile_run.finished_time is '完成时间'; +comment on column sys_file_reconcile_run.error_message is '失败原因'; + +-- ---------------------------- +-- 29、文件存储对账异常表 +-- ---------------------------- +drop table if exists sys_file_reconcile_issue; +create table sys_file_reconcile_issue ( + issue_id bigserial not null, + issue_key varchar(64) not null, + last_run_id varchar(36) not null, + issue_type varchar(32) not null, + severity varchar(10) not null, + file_id varchar(36), + storage_type varchar(20), + access_type varchar(20), + expected_root varchar(20), + expected_key varchar(500), + actual_root varchar(20), + actual_key varchar(500), + expected_size bigint, + actual_size bigint, + expected_hash varchar(64), + actual_hash varchar(64), + status varchar(20) not null default 'open', + detail text, + occurrence_count integer not null default 1, + first_seen_time timestamp(0) not null, + last_seen_time timestamp(0) not null, + handle_action varchar(32), + handle_reason varchar(500), + handled_by varchar(64), + handled_time timestamp(0), + quarantine_key varchar(500), + primary key (issue_id) +); +create unique index uk_sys_file_reconcile_issue_key on sys_file_reconcile_issue(issue_key); +create index idx_sys_file_reconcile_issue_status_severity on sys_file_reconcile_issue(status, severity); +create index idx_sys_file_reconcile_issue_file on sys_file_reconcile_issue(file_id); +create index idx_sys_file_reconcile_issue_run on sys_file_reconcile_issue(last_run_id); +comment on table sys_file_reconcile_issue is '文件存储对账异常表'; +comment on column sys_file_reconcile_issue.issue_id is '异常ID'; +comment on column sys_file_reconcile_issue.issue_key is '异常唯一标识'; +comment on column sys_file_reconcile_issue.last_run_id is '最近发现任务ID'; +comment on column sys_file_reconcile_issue.issue_type is '异常类型'; +comment on column sys_file_reconcile_issue.severity is '严重级别'; +comment on column sys_file_reconcile_issue.file_id is '文件ID'; +comment on column sys_file_reconcile_issue.storage_type is '存储类型'; +comment on column sys_file_reconcile_issue.access_type is '访问类型'; +comment on column sys_file_reconcile_issue.expected_root is '预期存储区域'; +comment on column sys_file_reconcile_issue.expected_key is '预期相对路径'; +comment on column sys_file_reconcile_issue.actual_root is '实际存储区域'; +comment on column sys_file_reconcile_issue.actual_key is '实际相对路径'; +comment on column sys_file_reconcile_issue.expected_size is '预期文件大小'; +comment on column sys_file_reconcile_issue.actual_size is '实际文件大小'; +comment on column sys_file_reconcile_issue.expected_hash is '预期SHA-256'; +comment on column sys_file_reconcile_issue.actual_hash is '实际SHA-256'; +comment on column sys_file_reconcile_issue.status is '处理状态'; +comment on column sys_file_reconcile_issue.detail is '异常说明'; +comment on column sys_file_reconcile_issue.occurrence_count is '发现次数'; +comment on column sys_file_reconcile_issue.first_seen_time is '首次发现时间'; +comment on column sys_file_reconcile_issue.last_seen_time is '最近发现时间'; +comment on column sys_file_reconcile_issue.handle_action is '处理动作'; +comment on column sys_file_reconcile_issue.handle_reason is '处理原因'; +comment on column sys_file_reconcile_issue.handled_by is '处理人'; +comment on column sys_file_reconcile_issue.handled_time is '处理时间'; +comment on column sys_file_reconcile_issue.quarantine_key is '隔离区相对路径'; + CREATE OR REPLACE FUNCTION "find_in_set"(int8, varchar) RETURNS "pg_catalog"."bool" AS $BODY$ DECLARE diff --git a/ruoyi-fastapi-backend/sql/ruoyi-fastapi.sql b/ruoyi-fastapi-backend/sql/ruoyi-fastapi.sql index 623b9bf..c2d3468 100644 --- a/ruoyi-fastapi-backend/sql/ruoyi-fastapi.sql +++ b/ruoyi-fastapi-backend/sql/ruoyi-fastapi.sql @@ -174,6 +174,7 @@ insert into sys_menu values('105', '字典管理', '1', '6', 'dict', insert into sys_menu values('106', '参数设置', '1', '7', 'config', 'system/config/index', '', '', 1, 0, 'C', '0', '0', 'system:config:list', 'edit', 'admin', sysdate(), '', null, '参数设置菜单'); insert into sys_menu values('107', '通知公告', '1', '8', 'notice', 'system/notice/index', '', '', 1, 0, 'C', '0', '0', 'system:notice:list', 'message', 'admin', sysdate(), '', null, '通知公告菜单'); insert into sys_menu values('108', '日志管理', '1', '9', 'log', '', '', '', 1, 0, 'M', '0', '0', '', 'log', 'admin', sysdate(), '', null, '日志管理菜单'); +insert into sys_menu values('121', '文件管理', '1', '10', 'file', 'system/file/index', '', '', 1, 0, 'C', '0', '0', 'system:file:list', 'documentation', 'admin', sysdate(), '', null, '文件管理菜单'); insert into sys_menu values('109', '在线用户', '2', '1', 'online', 'monitor/online/index', '', '', 1, 0, 'C', '0', '0', 'monitor:online:list', 'online', 'admin', sysdate(), '', null, '在线用户菜单'); insert into sys_menu values('110', '定时任务', '2', '2', 'job', 'monitor/job/index', '', '', 1, 0, 'C', '0', '0', 'monitor:job:list', 'job', 'admin', sysdate(), '', null, '定时任务菜单'); insert into sys_menu values('111', '数据监控', '2', '3', 'druid', 'monitor/druid/index', '', '', 1, 0, 'C', '0', '0', 'monitor:druid:list', 'druid', 'admin', sysdate(), '', null, '数据监控菜单'); @@ -236,6 +237,15 @@ insert into sys_menu values('1035', '公告查询', '107', '1', '#', '', '', '', insert into sys_menu values('1036', '公告新增', '107', '2', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:notice:add', '#', 'admin', sysdate(), '', null, ''); insert into sys_menu values('1037', '公告修改', '107', '3', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:notice:edit', '#', 'admin', sysdate(), '', null, ''); insert into sys_menu values('1038', '公告删除', '107', '4', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:notice:remove', '#', 'admin', sysdate(), '', null, ''); +-- 文件管理按钮 +insert into sys_menu values('1065', '文件查询', '121', '1', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:file:query', '#', 'admin', sysdate(), '', null, ''); +insert into sys_menu values('1066', '文件下载', '121', '2', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:file:download', '#', 'admin', sysdate(), '', null, ''); +insert into sys_menu values('1067', '文件删除', '121', '3', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:file:remove', '#', 'admin', sysdate(), '', null, ''); +insert into sys_menu values('1068', '文件授权', '121', '4', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:file:edit', '#', 'admin', sysdate(), '', null, ''); +insert into sys_menu values('1069', '文件转移', '121', '5', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:file:transfer', '#', 'admin', sysdate(), '', null, ''); +insert into sys_menu values('1070', '文件恢复', '121', '6', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:file:restore', '#', 'admin', sysdate(), '', null, ''); +insert into sys_menu values('1071', '文件清理', '121', '7', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:file:purge', '#', 'admin', sysdate(), '', null, ''); +insert into sys_menu values('1072', '存储对账', '121', '8', '#', '', '', '', 1, 0, 'F', '0', '0', 'system:file:reconcile', '#', 'admin', sysdate(), '', null, ''); -- 操作日志按钮 insert into sys_menu values('1039', '操作查询', '500', '1', '#', '', '', '', 1, 0, 'F', '0', '0', 'monitor:operlog:query', '#', 'admin', sysdate(), '', null, ''); insert into sys_menu values('1040', '操作删除', '500', '2', '#', '', '', '', 1, 0, 'F', '0', '0', 'monitor:operlog:remove', '#', 'admin', sysdate(), '', null, ''); @@ -654,6 +664,9 @@ create table sys_job ( insert into sys_job values(1, '系统默认（无参）', 'default', 'default', 'module_task.scheduler_test.job', NULL, NULL, '0/10 * * * * ?', '3', '1', '1', 'admin', sysdate(), '', null, ''); insert into sys_job values(2, '系统默认（有参）', 'default', 'default', 'module_task.scheduler_test.job', 'test', NULL, '0/15 * * * * ?', '3', '1', '1', 'admin', sysdate(), '', null, ''); insert into sys_job values(3, '系统默认（多参）', 'default', 'default', 'module_task.scheduler_test.job', 'new', '{\"test\": 111}', '0/20 * * * * ?', '3', '1', '1', 'admin', sysdate(), '', null, ''); +insert into sys_job values(4, '文件保留期限提醒', 'default', 'default', 'module_task.file_task.scan_retention_reminders', NULL, '{\"remind_days\": 7, \"batch_size\": 500}', '0 0 1 * * ?', '3', '1', '0', 'admin', sysdate(), '', null, '每天扫描即将到期和已到期的受保护文件'); +insert into sys_job values(5, '回收站永久清理', 'default', 'default', 'module_task.file_task.purge_recycle_bin', NULL, '{\"retention_days\": 30, \"batch_size\": 100}', '0 0 2 * * ?', '3', '1', '1', 'admin', sysdate(), '', null, '永久清理超过保留期限的回收站文件，默认暂停'); +insert into sys_job values(6, '文件存储对账', 'default', 'default', 'module_task.file_task.reconcile_file_storage', NULL, '{\"check_hash\": false}', '0 0 3 * * ?', '3', '1', '1', 'admin', sysdate(), '', null, '校验文件信息表和本地存储一致性，默认暂停'); -- ---------------------------- @@ -811,3 +824,211 @@ create table ai_chat_config ( update_time datetime comment '更新时间', primary key (chat_config_id) ) engine=innodb auto_increment=1 comment = 'AI对话配置表'; + + +-- ---------------------------- +-- 22、文件信息表 +-- ---------------------------- +drop table if exists sys_file_info; +create table sys_file_info ( + file_id varchar(36) not null comment '文件ID', + original_name varchar(255) not null comment '原始文件名', + stored_name varchar(255) not null comment '存储文件名', + storage_key varchar(500) not null comment '存储相对路径', + storage_type varchar(20) not null default 'local' comment '存储类型', + access_type varchar(20) not null default 'public' comment '访问类型', + upload_user_id bigint(20) comment '上传用户ID', + uploader_access_enabled char(1) not null default '1' comment '是否保留上传人访问权限', + owner_user_id bigint(20) comment '所有者用户ID', + dept_id bigint(20) comment '所属部门ID', + acl_version int not null default 0 comment '访问控制版本', + business_type varchar(50) comment '业务类型', + business_id varchar(64) comment '业务ID', + extension varchar(20) not null default '' comment '文件扩展名', + content_type varchar(255) comment '内容类型', + file_size bigint(20) not null default 0 comment '文件大小', + file_hash varchar(64) not null comment '文件SHA-256', + status varchar(20) not null default 'active' comment '文件状态', + create_by varchar(64) default '' comment '创建者', + create_time datetime not null comment '创建时间', + update_by varchar(64) default '' comment '更新者', + update_time datetime not null comment '更新时间', + expire_time datetime comment '过期时间', + deleted_time datetime comment '移入回收站时间', + del_flag char(1) not null default '0' comment '删除标志', + primary key (file_id), + unique key uk_sys_file_info_storage_location (storage_type, access_type, storage_key), + key idx_sys_file_info_access_status (access_type, status), + key idx_sys_file_info_owner_status (owner_user_id, status), + key idx_sys_file_info_dept_status (dept_id, status), + key idx_sys_file_info_status_deleted_time (status, deleted_time) +) engine=innodb comment = '文件信息表'; + + +-- ---------------------------- +-- 23、文件业务引用表 +-- ---------------------------- +drop table if exists sys_file_reference; +create table sys_file_reference ( + reference_id bigint(20) not null auto_increment comment '引用ID', + file_id varchar(36) not null comment '文件ID', + business_type varchar(50) not null comment '业务类型', + business_id varchar(64) not null comment '业务ID', + business_name varchar(255) comment '业务名称', + retention_expire_time datetime comment '保留期限到期时间', + create_by varchar(64) default '' comment '创建者', + create_time datetime not null comment '创建时间', + primary key (reference_id), + unique key uk_sys_file_reference_business (file_id, business_type, business_id), + key idx_sys_file_reference_file (file_id), + key idx_sys_file_reference_business (business_type, business_id) +) engine=innodb auto_increment=1 comment = '文件业务引用表'; + + +-- ---------------------------- +-- 24、文件业务保留策略表 +-- ---------------------------- +drop table if exists sys_file_retention_policy; +create table sys_file_retention_policy ( + business_type varchar(50) not null comment '业务类型', + retention_days int not null comment '保留天数', + status char(1) not null default '0' comment '状态（0启用 1停用）', + remark varchar(500) comment '备注', + create_by varchar(64) default '' comment '创建者', + create_time datetime not null comment '创建时间', + update_by varchar(64) default '' comment '更新者', + update_time datetime not null comment '更新时间', + primary key (business_type) +) engine=innodb comment = '文件业务保留策略表'; + + +-- ---------------------------- +-- 25、文件保留期限提醒表 +-- ---------------------------- +drop table if exists sys_file_retention_notice; +create table sys_file_retention_notice ( + notice_id bigint(20) not null auto_increment comment '提醒ID', + file_id varchar(36) not null comment '文件ID', + notice_type varchar(20) not null comment '提醒类型', + expire_time datetime not null comment '文件过期时间', + status char(1) not null default '0' comment '状态（0未读 1已读 2已失效）', + create_time datetime not null comment '创建时间', + read_by varchar(64) default '' comment '读取者', + read_time datetime comment '读取时间', + primary key (notice_id), + unique key uk_sys_file_retention_notice_file_type_time (file_id, notice_type, expire_time), + key idx_sys_file_retention_notice_file (file_id), + key idx_sys_file_retention_notice_status_time (status, create_time) +) engine=innodb auto_increment=1 comment = '文件保留期限提醒表'; + + +-- ---------------------------- +-- 26、文件访问控制表 +-- ---------------------------- +drop table if exists sys_file_acl; +create table sys_file_acl ( + acl_id bigint(20) not null auto_increment comment '访问控制ID', + file_id varchar(36) not null comment '文件ID', + subject_type varchar(20) not null comment '主体类型', + subject_id bigint(20) not null comment '主体ID', + permission varchar(20) not null default 'download' comment '权限类型', + effect varchar(10) not null default 'allow' comment '授权效果', + include_children char(1) not null default '0' comment '部门是否包含下级', + expire_time datetime comment '授权过期时间', + create_by varchar(64) default '' comment '创建者', + create_time datetime not null comment '创建时间', + del_flag char(1) not null default '0' comment '删除标志', + primary key (acl_id), + unique key uk_sys_file_acl_subject_permission (file_id, subject_type, subject_id, permission), + key idx_sys_file_acl_file_status (file_id, del_flag, expire_time), + key idx_sys_file_acl_subject (subject_type, subject_id) +) engine=innodb auto_increment=1 comment = '文件访问控制表'; + + +-- ---------------------------- +-- 27、文件访问审计表 +-- ---------------------------- +drop table if exists sys_file_access_log; +create table sys_file_access_log ( + audit_id bigint(20) not null auto_increment comment '审计ID', + file_id varchar(36) not null comment '文件ID', + action varchar(20) not null comment '操作类型', + actor_user_id bigint(20) comment '操作用户ID', + actor_name varchar(64) default '' comment '操作用户名称', + result varchar(20) not null comment '操作结果', + request_id varchar(64) default '' comment '请求ID', + trace_id varchar(64) default '' comment '链路ID', + ip_address varchar(128) default '' comment '客户端地址', + user_agent varchar(500) default '' comment '用户代理', + bytes_sent bigint(20) not null default 0 comment '发送字节数', + error_message varchar(500) default '' comment '失败原因', + operation_detail text comment '操作详情', + access_time datetime not null comment '访问时间', + primary key (audit_id), + key idx_sys_file_access_log_file_time (file_id, access_time), + key idx_sys_file_access_log_actor_time (actor_user_id, access_time) +) engine=innodb auto_increment=1 comment = '文件访问审计表'; + + +-- ---------------------------- +-- 28、文件存储对账任务表 +-- ---------------------------- +drop table if exists sys_file_reconcile_run; +create table sys_file_reconcile_run ( + run_id varchar(36) not null comment '任务ID', + trigger_type varchar(20) not null comment '触发类型', + status varchar(20) not null comment '任务状态', + check_hash char(1) not null default '0' comment '是否校验文件摘要', + lock_name varchar(32) comment '运行锁名称', + scanned_file_count bigint(20) not null default 0 comment '扫描文件记录数', + scanned_storage_count bigint(20) not null default 0 comment '扫描物理文件数', + issue_count bigint(20) not null default 0 comment '发现异常数', + new_issue_count bigint(20) not null default 0 comment '新增或重新出现异常数', + resolved_issue_count bigint(20) not null default 0 comment '自动恢复异常数', + started_by varchar(64) default '' comment '发起人', + started_time datetime not null comment '开始时间', + finished_time datetime comment '完成时间', + error_message text comment '失败原因', + primary key (run_id), + unique key uk_sys_file_reconcile_run_lock (lock_name), + key idx_sys_file_reconcile_run_status_time (status, started_time) +) engine=innodb comment = '文件存储对账任务表'; + + +-- ---------------------------- +-- 29、文件存储对账异常表 +-- ---------------------------- +drop table if exists sys_file_reconcile_issue; +create table sys_file_reconcile_issue ( + issue_id bigint(20) not null auto_increment comment '异常ID', + issue_key varchar(64) not null comment '异常唯一标识', + last_run_id varchar(36) not null comment '最近发现任务ID', + issue_type varchar(32) not null comment '异常类型', + severity varchar(10) not null comment '严重级别', + file_id varchar(36) comment '文件ID', + storage_type varchar(20) comment '存储类型', + access_type varchar(20) comment '访问类型', + expected_root varchar(20) comment '预期存储区域', + expected_key varchar(500) comment '预期相对路径', + actual_root varchar(20) comment '实际存储区域', + actual_key varchar(500) comment '实际相对路径', + expected_size bigint(20) comment '预期文件大小', + actual_size bigint(20) comment '实际文件大小', + expected_hash varchar(64) comment '预期SHA-256', + actual_hash varchar(64) comment '实际SHA-256', + status varchar(20) not null default 'open' comment '处理状态', + detail text comment '异常说明', + occurrence_count int(11) not null default 1 comment '发现次数', + first_seen_time datetime not null comment '首次发现时间', + last_seen_time datetime not null comment '最近发现时间', + handle_action varchar(32) comment '处理动作', + handle_reason varchar(500) comment '处理原因', + handled_by varchar(64) comment '处理人', + handled_time datetime comment '处理时间', + quarantine_key varchar(500) comment '隔离区相对路径', + primary key (issue_id), + unique key uk_sys_file_reconcile_issue_key (issue_key), + key idx_sys_file_reconcile_issue_status_severity (status, severity), + key idx_sys_file_reconcile_issue_file (file_id), + key idx_sys_file_reconcile_issue_run (last_run_id) +) engine=innodb auto_increment=1 comment = '文件存储对账异常表'; diff --git a/ruoyi-fastapi-backend/sub_applications/staticfiles.py b/ruoyi-fastapi-backend/sub_applications/staticfiles.py index 5e5d4e5..7d99031 100644 --- a/ruoyi-fastapi-backend/sub_applications/staticfiles.py +++ b/ruoyi-fastapi-backend/sub_applications/staticfiles.py @@ -1,11 +1,48 @@ +from pathlib import Path +from urllib.parse import quote + from fastapi import FastAPI from fastapi.staticfiles import StaticFiles +from starlette.responses import Response +from starlette.types import Scope from config.env import UploadConfig +class SecureStaticFiles(StaticFiles): + """ + 安全静态文件服务类 + """ + + DOWNLOAD_ONLY_EXTENSIONS = {'.html', '.htm'} + + async def get_response(self, path: str, scope: Scope) -> Response: + """ + 获取带有安全响应头的静态文件响应 + + :param path: 静态文件路径 + :param scope: ASGI连接作用域 + :return: 静态文件响应 + """ + response = await super().get_response(path, scope) + response.headers['X-Content-Type-Options'] = 'nosniff' + if Path(path).suffix.lower() in self.DOWNLOAD_ONLY_EXTENSIONS: + encoded_name = quote(Path(path).name) + response.headers['Content-Type'] = 'application/octet-stream' + response.headers['Content-Disposition'] = f"attachment; filename*=UTF-8''{encoded_name}" + response.headers['Content-Security-Policy'] = ( + "sandbox; default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" + ) + response.headers['X-Frame-Options'] = 'DENY' + return response + + def mount_staticfiles(app: FastAPI) -> None: """ 挂载静态文件 """ - app.mount(f'{UploadConfig.UPLOAD_PREFIX}', StaticFiles(directory=f'{UploadConfig.UPLOAD_PATH}'), name='profile') + app.mount( + f'{UploadConfig.UPLOAD_PREFIX}', + SecureStaticFiles(directory=f'{UploadConfig.UPLOAD_PATH}'), + name='profile', + ) diff --git a/ruoyi-fastapi-backend/tests/test_common_file_security.py b/ruoyi-fastapi-backend/tests/test_common_file_security.py new file mode 100644 index 0000000..c958cb2 --- /dev/null +++ b/ruoyi-fastapi-backend/tests/test_common_file_security.py @@ -0,0 +1,721 @@ +import asyncio +import io +import os +import re +import sys +from collections.abc import AsyncGenerator +from datetime import datetime, timedelta +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest +from fastapi import BackgroundTasks, FastAPI, UploadFile +from fastapi.middleware.cors import CORSMiddleware +from sqlalchemy import false + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +from config.env import UploadConfig +from exceptions.exception import FileRangeNotSatisfiableException, ServiceException +from middlewares.cors_middleware import add_cors_middleware +from module_admin.dao.file_access_dao import FileAclDao +from module_admin.dao.file_info_dao import FileInfoDao +from module_admin.service.common_service import CommonService +from sub_applications.staticfiles import SecureStaticFiles +from utils.file_util import FileUtil +from utils.upload_util import FilePathUtil, UploadUtil + +RANGE_TEST_FILE_SIZE = 10 +RANGE_TEST_START = 3 +RANGE_TEST_END = 6 +RANGE_TEST_LENGTH = RANGE_TEST_END - RANGE_TEST_START + 1 + + +async def collect_stream(stream: AsyncGenerator[bytes, None]) -> bytes: + return b''.join([chunk async for chunk in stream]) + + +async def collect_first_chunk_and_close(stream: AsyncGenerator[bytes, None]) -> bytes: + chunk = await anext(stream) + await stream.aclose() + return chunk + + +def make_upload(filename: str, content: bytes) -> UploadFile: + return UploadFile(file=io.BytesIO(content), filename=filename) + + +def make_current_user(user_id: int = 10, admin: bool = False, dept_id: int = 100) -> SimpleNamespace: + return SimpleNamespace( + user=SimpleNamespace(user_id=user_id, user_name=f'user{user_id}', admin=admin, dept_id=dept_id) + ) + + +def make_query_db() -> SimpleNamespace: + return SimpleNamespace(commit=AsyncMock(), rollback=AsyncMock()) + + +@pytest.mark.parametrize( + ('range_header', 'expected'), + [ + (None, (0, 9, 10, False, 10)), + ('bytes=2-5', (2, 5, 10, True, 4)), + ('bytes=7-', (7, 9, 10, True, 3)), + ('bytes=-3', (7, 9, 10, True, 3)), + ('bytes=-99', (0, 9, 10, True, 10)), + ('bytes=3-99', (3, 9, 10, True, 7)), + ], +) +def test_parse_byte_range_supports_standard_single_ranges( + range_header: str | None, + expected: tuple[int, int, int, bool, int], +) -> None: + byte_range = FileUtil.parse_byte_range(range_header, 10) + + assert ( + byte_range.start, + byte_range.end, + byte_range.file_size, + byte_range.is_partial, + byte_range.length, + ) == expected + + +@pytest.mark.parametrize( + 'range_header', + [ + 'items=0-1', + 'bytes=', + 'bytes=-0', + 'bytes=10-', + 'bytes=8-7', + 'bytes=0-1,3-4', + ], +) +def test_parse_byte_range_rejects_invalid_or_multiple_ranges(range_header: str) -> None: + with pytest.raises(FileRangeNotSatisfiableException) as range_error: + FileUtil.parse_byte_range(range_header, RANGE_TEST_FILE_SIZE) + + assert range_error.value.file_size == RANGE_TEST_FILE_SIZE + + +def test_parse_byte_range_allows_empty_full_download_but_rejects_empty_partial_download() -> None: + byte_range = FileUtil.parse_byte_range(None, 0) + + assert byte_range.length == 0 + assert byte_range.is_partial is False + with pytest.raises(FileRangeNotSatisfiableException): + FileUtil.parse_byte_range('bytes=0-', 0) + + +def test_resolve_file_within_root_accepts_nested_relative_file(tmp_path: Path) -> None: + root = tmp_path / 'download' + target = root / 'nested' / 'report.txt' + target.parent.mkdir(parents=True) + target.write_text('safe', encoding='utf-8') + + resolved = FilePathUtil.resolve_file_within_root(root, 'nested/report.txt') + + assert resolved == target.resolve() + + +@pytest.mark.parametrize( + 'untrusted_path', + [ + '/etc/passwd', + 'C:\\Windows\\win.ini', + '\\\\server\\share\\secret.txt', + '../secret.txt', + '..\\secret.txt', + 'nested/../secret.txt', + 'nested\\..\\secret.txt', + ], +) +def test_resolve_file_within_root_rejects_absolute_and_traversal_paths(tmp_path: Path, untrusted_path: str) -> None: + root = tmp_path / 'download' + root.mkdir() + + with pytest.raises(ValueError): + FilePathUtil.resolve_file_within_root(root, untrusted_path) + + +def test_resolve_file_within_root_rejects_symlink_escape(tmp_path: Path) -> None: + root = tmp_path / 'download' + root.mkdir() + outside = tmp_path / 'outside.txt' + outside.write_text('secret', encoding='utf-8') + link = root / 'link.txt' + try: + link.symlink_to(outside) + except OSError: + pytest.skip('当前环境不允许创建符号链接') + + with pytest.raises(ValueError): + FilePathUtil.resolve_file_within_root(root, 'link.txt') + + +def test_download_rejects_absolute_path_without_scheduling_delete(tmp_path: Path) -> None: + download_root = tmp_path / 'download' + download_root.mkdir() + outside = tmp_path / 'outside.txt' + outside.write_text('secret', encoding='utf-8') + background_tasks = BackgroundTasks() + + with ( + patch.object(UploadConfig, 'DOWNLOAD_PATH', str(download_root)), + pytest.raises(ServiceException), + ): + asyncio.run(CommonService.download_services(background_tasks, str(outside.resolve()), True)) + + assert background_tasks.tasks == [] + assert outside.exists() + + +def test_download_keeps_delete_compatibility_inside_download_root(tmp_path: Path) -> None: + download_root = tmp_path / 'download' + download_root.mkdir() + target = download_root / 'report.txt' + target.write_bytes(b'report-content') + background_tasks = BackgroundTasks() + + with patch.object(UploadConfig, 'DOWNLOAD_PATH', str(download_root)): + result = asyncio.run( + CommonService.download_services( + background_tasks, + 'report.txt', + True, + range_header='bytes=0-5', + ) + ) + assert asyncio.run(collect_stream(result.data)) == b'report-content' + assert result.byte_range.is_partial is False + assert result.accept_ranges is False + assert target.exists() + asyncio.run(background_tasks()) + + assert not target.exists() + + +def test_resource_download_is_confined_to_upload_root(tmp_path: Path) -> None: + upload_root = tmp_path / 'profile' + target = upload_root / 'upload' / '2026' / '07' / 'report_20260719120000A001.txt' + target.parent.mkdir(parents=True) + target.write_bytes(b'resource-content') + + with patch.object(UploadConfig, 'UPLOAD_PATH', str(upload_root)): + result = asyncio.run( + CommonService.download_resource_services('/profile/upload/2026/07/report_20260719120000A001.txt') + ) + assert asyncio.run(collect_stream(result.data)) == b'resource-content' + + range_result = asyncio.run( + CommonService.download_resource_services( + '/profile/upload/2026/07/report_20260719120000A001.txt', + range_header='bytes=3-10', + ) + ) + assert asyncio.run(collect_stream(range_result.data)) == b'ource-co' + assert range_result.byte_range.is_partial is True + + with pytest.raises(ServiceException): + asyncio.run(CommonService.download_resource_services('/profile/../outside.txt')) + + +def test_resource_download_rejects_file_without_generated_filename(tmp_path: Path) -> None: + upload_root = tmp_path / 'profile' + target = upload_root / 'upload' / '2026' / '07' / 'report.txt' + target.parent.mkdir(parents=True) + target.write_bytes(b'resource-content') + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(upload_root)), + pytest.raises(ServiceException), + ): + asyncio.run(CommonService.download_resource_services('/profile/upload/2026/07/report.txt')) + + +def test_upload_uses_server_generated_filename_and_stays_in_upload_root(tmp_path: Path) -> None: + upload_root = tmp_path / 'profile' + upload = make_upload('../../attack:bad?.txt', b'safe-content') + request = SimpleNamespace(base_url='https://example.test/prod-api/', headers={}, client=None) + query_db = make_query_db() + current_user = make_current_user() + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(upload_root)), + patch.object(UploadConfig, 'MAX_FILE_SIZE', 1024), + patch.object(FileInfoDao, 'add_file_info_dao', new_callable=AsyncMock) as add_file_info, + patch.object(CommonService, '_enqueue_file_access_log', new_callable=AsyncMock), + ): + result = asyncio.run(CommonService.upload_service(request, query_db, current_user, upload)) + + written_files = list(upload_root.rglob('*.*')) + assert len(written_files) == 1 + assert written_files[0].resolve().is_relative_to(upload_root.resolve()) + assert written_files[0].read_bytes() == b'safe-content' + assert result.result.original_filename == 'attack:bad?.txt' + assert re.fullmatch(r'attack_bad_\d{14}A\d{3}\.txt', result.result.new_file_name) + assert written_files[0].name == result.result.new_file_name + assert result.result.access_type == 'public' + assert result.result.file_id + file_info = add_file_info.await_args.args[1] + assert file_info.upload_user_id == current_user.user.user_id + assert file_info.owner_user_id == current_user.user.user_id + assert file_info.dept_id == current_user.user.dept_id + assert file_info.file_size == len(b'safe-content') + assert file_info.file_hash == '63a2f0f94f2efe262dee71613926b2bb5ceda47b0aa2950d9403dcfd5a089ec8' + query_db.commit.assert_awaited_once() + + with patch.object(UploadConfig, 'UPLOAD_PATH', str(upload_root)): + download_result = asyncio.run(CommonService.download_resource_services(result.result.file_name)) + assert asyncio.run(collect_stream(download_result.data)) == b'safe-content' + + +@pytest.mark.parametrize('extension', ['html', 'htm']) +def test_upload_accepts_html_as_download_only_file(tmp_path: Path, extension: str) -> None: + file_content = b'' + upload = make_upload(f'attack.{extension}', file_content) + request = SimpleNamespace(base_url='https://example.test/prod-api/', headers={}, client=None) + query_db = make_query_db() + current_user = make_current_user() + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(tmp_path)), + patch.object(UploadConfig, 'MAX_FILE_SIZE', 1024), + patch.object(FileInfoDao, 'add_file_info_dao', new_callable=AsyncMock), + patch.object(CommonService, '_enqueue_file_access_log', new_callable=AsyncMock), + ): + result = asyncio.run(CommonService.upload_service(request, query_db, current_user, upload)) + download_result = asyncio.run(CommonService.download_resource_services(result.result.file_name)) + + written_files = [path for path in tmp_path.rglob('*') if path.is_file()] + assert len(written_files) == 1 + assert written_files[0].read_bytes() == file_content + assert re.fullmatch(rf'attack_\d{{14}}A\d{{3}}\.{extension}', result.result.new_file_name) + assert asyncio.run(collect_stream(download_result.data)) == file_content + + +def test_upload_enforces_total_size_and_removes_partial_file(tmp_path: Path) -> None: + upload = make_upload('large.txt', b'12345') + request = SimpleNamespace(base_url='https://example.test/prod-api/', headers={}, client=None) + query_db = make_query_db() + current_user = make_current_user() + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(tmp_path)), + patch.object(UploadConfig, 'MAX_FILE_SIZE', 4), + pytest.raises(ServiceException), + ): + asyncio.run(CommonService.upload_service(request, query_db, current_user, upload)) + + assert [path for path in tmp_path.rglob('*') if path.is_file()] == [] + + +def test_upload_rejects_unknown_access_type_before_writing(tmp_path: Path) -> None: + upload = make_upload('report.txt', b'report-content') + request = SimpleNamespace(base_url='https://example.test/prod-api/', headers={}, client=None) + + with ( + patch.object(UploadConfig, 'PRIVATE_UPLOAD_PATH', str(tmp_path)), + pytest.raises(ServiceException), + ): + asyncio.run(CommonService.upload_service(request, make_query_db(), make_current_user(), upload, 'shared')) + + assert list(tmp_path.rglob('*')) == [] + + +def test_upload_removes_file_when_metadata_write_fails(tmp_path: Path) -> None: + upload = make_upload('report.txt', b'report-content') + request = SimpleNamespace(base_url='https://example.test/prod-api/', headers={}, client=None) + query_db = make_query_db() + current_user = make_current_user() + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(tmp_path)), + patch.object(FileInfoDao, 'add_file_info_dao', new=AsyncMock(side_effect=RuntimeError('db error'))), + patch.object(CommonService, '_enqueue_file_access_log', new_callable=AsyncMock), + pytest.raises(RuntimeError), + ): + asyncio.run(CommonService.upload_service(request, query_db, current_user, upload)) + + assert [path for path in tmp_path.rglob('*') if path.is_file()] == [] + query_db.rollback.assert_awaited_once() + + +def test_upload_retries_name_collision_without_deleting_existing_file(tmp_path: Path) -> None: + fixed_time = datetime(2026, 7, 19, 12, 0, 0) + existing_file = tmp_path / 'upload' / '2026' / '07' / '19' / 'report_20260719120000A001.txt' + existing_file.parent.mkdir(parents=True) + existing_file.write_bytes(b'existing-content') + upload = make_upload('report.txt', b'new-content') + request = SimpleNamespace(base_url='https://example.test/prod-api/', headers={}, client=None) + query_db = make_query_db() + current_user = make_current_user() + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(tmp_path)), + patch('module_admin.service.common_service.datetime', new=SimpleNamespace(now=lambda: fixed_time)), + patch.object(UploadUtil, 'generate_random_number', side_effect=['001', '002']), + patch.object(FileInfoDao, 'add_file_info_dao', new_callable=AsyncMock), + patch.object(CommonService, '_enqueue_file_access_log', new_callable=AsyncMock), + ): + result = asyncio.run(CommonService.upload_service(request, query_db, current_user, upload)) + + assert existing_file.read_bytes() == b'existing-content' + assert result.result.new_file_name == 'report_20260719120000A002.txt' + assert (existing_file.parent / result.result.new_file_name).read_bytes() == b'new-content' + + +def test_private_upload_is_physically_isolated_and_owner_can_download(tmp_path: Path) -> None: + public_root = tmp_path / 'public' + private_root = tmp_path / 'private' + upload = make_upload('contract.pdf', b'private-content') + request = SimpleNamespace(base_url='https://example.test/prod-api/', headers={}, client=None) + query_db = make_query_db() + current_user = make_current_user() + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(public_root)), + patch.object(UploadConfig, 'PRIVATE_UPLOAD_PATH', str(private_root)), + patch.object(FileInfoDao, 'add_file_info_dao', new_callable=AsyncMock) as add_file_info, + patch.object(CommonService, '_enqueue_file_access_log', new_callable=AsyncMock) as enqueue_audit, + ): + result = asyncio.run( + CommonService.upload_service(request, query_db, current_user, upload, access_type='private') + ) + file_info = add_file_info.await_args.args[1] + with patch.object(FileInfoDao, 'get_file_info_by_id', new=AsyncMock(return_value=file_info)): + download_result = asyncio.run( + CommonService.download_managed_file_services(request, query_db, current_user, result.result.file_id) + ) + assert asyncio.run(collect_stream(download_result.data)) == b'private-content' + + assert list(public_root.rglob('*')) == [] + assert len([path for path in private_root.rglob('*') if path.is_file()]) == 1 + assert result.result.file_name.startswith('/common/files/') + assert not result.result.file_name.startswith(UploadConfig.UPLOAD_PREFIX) + assert result.result.access_type == 'private' + assert download_result.filename == 'contract.pdf' + audit_results = [call.kwargs['result'] for call in enqueue_audit.await_args_list] + assert audit_results == ['completed', 'allowed', 'completed'] + + +def test_private_download_rejects_other_user(tmp_path: Path) -> None: + private_root = tmp_path / 'private' + target = private_root / 'upload' / '2026' / '07' / 'contract_20260719120000A001.pdf' + target.parent.mkdir(parents=True) + target.write_bytes(b'private-content') + file_info = SimpleNamespace( + storage_type='local', + access_type='private', + upload_user_id=10, + owner_user_id=10, + expire_time=None, + storage_key='upload/2026/07/contract_20260719120000A001.pdf', + original_name='contract.pdf', + ) + request = SimpleNamespace(base_url='https://example.test/prod-api/', headers={}, client=None) + query_db = make_query_db() + other_user = make_current_user(user_id=20) + + with ( + patch.object(UploadConfig, 'PRIVATE_UPLOAD_PATH', str(private_root)), + patch.object(FileInfoDao, 'get_file_info_by_id', new=AsyncMock(return_value=file_info)), + patch.object(FileAclDao, 'get_effective_file_acl_list', new=AsyncMock(return_value=[])), + patch.object(CommonService, '_enqueue_file_access_log', new_callable=AsyncMock) as enqueue_audit, + pytest.raises(ServiceException), + ): + asyncio.run(CommonService.download_managed_file_services(request, query_db, other_user, 'file-id')) + + enqueue_audit.assert_awaited_once() + assert enqueue_audit.await_args.kwargs['result'] == 'denied' + + +def test_file_manager_can_download_private_file_without_owner_match(tmp_path: Path) -> None: + private_root = tmp_path / 'private' + target = private_root / 'upload' / '2026' / '07' / 'contract_20260719120000A001.pdf' + target.parent.mkdir(parents=True) + target.write_bytes(b'private-content') + file_info = SimpleNamespace( + storage_type='local', + access_type='private', + upload_user_id=10, + owner_user_id=10, + expire_time=None, + storage_key='upload/2026/07/contract_20260719120000A001.pdf', + original_name='contract.pdf', + ) + request = SimpleNamespace(base_url='https://example.test/prod-api/', headers={}, client=None) + query_db = make_query_db() + file_manager = make_current_user(user_id=20) + + with ( + patch.object(UploadConfig, 'PRIVATE_UPLOAD_PATH', str(private_root)), + patch.object(FileInfoDao, 'get_file_info_by_id', new=AsyncMock(return_value=file_info)), + patch.object(CommonService, '_enqueue_file_access_log', new_callable=AsyncMock), + ): + download_result = asyncio.run( + CommonService.download_managed_file_services( + request, + query_db, + file_manager, + 'file-id', + enforce_owner_permission=False, + ) + ) + assert asyncio.run(collect_stream(download_result.data)) == b'private-content' + + assert download_result.filename == 'contract.pdf' + + +def test_file_manager_download_rejects_file_outside_data_scope() -> None: + request = SimpleNamespace(base_url='https://example.test/prod-api/', headers={}, client=None) + query_db = make_query_db() + file_manager = make_current_user(user_id=20) + file_data_scope_sql = false() + + with ( + patch.object(FileInfoDao, 'get_file_info_by_id', new=AsyncMock(return_value=None)) as get_file_info, + patch.object(CommonService, '_enqueue_file_access_log', new_callable=AsyncMock) as enqueue_audit, + pytest.raises(ServiceException), + ): + asyncio.run( + CommonService.download_managed_file_services( + request, + query_db, + file_manager, + 'file-id', + enforce_owner_permission=False, + file_data_scope_sql=file_data_scope_sql, + ) + ) + + assert get_file_info.await_args.args[2] is file_data_scope_sql + assert enqueue_audit.await_args.kwargs['result'] == 'denied' + + +@pytest.mark.parametrize( + ('access_type', 'current_user'), + [ + ('private', make_current_user(user_id=1, admin=True)), + ('public', make_current_user(user_id=20)), + ], +) +def test_managed_download_allows_administrator_or_public_file( + tmp_path: Path, + access_type: str, + current_user: SimpleNamespace, +) -> None: + storage_root = tmp_path / access_type + target = storage_root / 'upload' / '2026' / '07' / 'report_20260719120000A001.txt' + target.parent.mkdir(parents=True) + target.write_bytes(b'file-content') + file_info = SimpleNamespace( + storage_type='local', + access_type=access_type, + upload_user_id=10, + owner_user_id=10, + expire_time=None, + storage_key='upload/2026/07/report_20260719120000A001.txt', + original_name='report.txt', + ) + request = SimpleNamespace(base_url='https://example.test/prod-api/', headers={}, client=None) + query_db = make_query_db() + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(storage_root)), + patch.object(UploadConfig, 'PRIVATE_UPLOAD_PATH', str(storage_root)), + patch.object(FileInfoDao, 'get_file_info_by_id', new=AsyncMock(return_value=file_info)), + patch.object(CommonService, '_enqueue_file_access_log', new_callable=AsyncMock) as enqueue_audit, + ): + download_result = asyncio.run( + CommonService.download_managed_file_services(request, query_db, current_user, 'file-id') + ) + assert asyncio.run(collect_stream(download_result.data)) == b'file-content' + + assert download_result.filename == 'report.txt' + assert [call.kwargs['result'] for call in enqueue_audit.await_args_list] == ['allowed', 'completed'] + + +@pytest.mark.parametrize('admin', [False, True]) +def test_private_download_rejects_expired_file(tmp_path: Path, admin: bool) -> None: + private_root = tmp_path / 'private' + file_info = SimpleNamespace( + storage_type='local', + access_type='private', + upload_user_id=10, + owner_user_id=10, + expire_time=datetime.now() - timedelta(seconds=1), + storage_key='upload/2026/07/report_20260719120000A001.txt', + original_name='report.txt', + ) + request = SimpleNamespace(base_url='https://example.test/prod-api/', headers={}, client=None) + query_db = make_query_db() + current_user = make_current_user(user_id=10, admin=admin) + + with ( + patch.object(UploadConfig, 'PRIVATE_UPLOAD_PATH', str(private_root)), + patch.object(FileInfoDao, 'get_file_info_by_id', new=AsyncMock(return_value=file_info)), + patch.object(CommonService, '_enqueue_file_access_log', new_callable=AsyncMock) as enqueue_audit, + pytest.raises(ServiceException), + ): + asyncio.run(CommonService.download_managed_file_services(request, query_db, current_user, 'file-id')) + + enqueue_audit.assert_awaited_once() + assert enqueue_audit.await_args.kwargs['result'] == 'denied' + + +def test_managed_download_records_interrupted_stream(tmp_path: Path) -> None: + target = tmp_path / 'report.txt' + target.write_bytes(b'partial-content') + request = SimpleNamespace(base_url='https://example.test/prod-api/', headers={}, client=None) + current_user = make_current_user() + + with patch.object(CommonService, '_enqueue_file_access_log', new_callable=AsyncMock) as enqueue_audit: + byte_range = FileUtil.parse_byte_range(None, target.stat().st_size) + stream = CommonService._generate_audited_file(request, current_user, 'file-id', target, byte_range) + assert asyncio.run(collect_first_chunk_and_close(stream)) == b'partial-content' + + enqueue_audit.assert_awaited_once() + assert enqueue_audit.await_args.kwargs['result'] == 'failed' + assert enqueue_audit.await_args.kwargs['error_message'] == 'StreamClosed' + + +def test_managed_download_supports_range_and_records_partial_bytes(tmp_path: Path) -> None: + storage_root = tmp_path / 'private' + target = storage_root / 'upload' / '2026' / '07' / 'report_20260719120000A001.txt' + target.parent.mkdir(parents=True) + target.write_bytes(b'0123456789') + file_info = SimpleNamespace( + storage_type='local', + access_type='private', + upload_user_id=10, + owner_user_id=10, + expire_time=None, + storage_key='upload/2026/07/report_20260719120000A001.txt', + original_name='report.txt', + ) + request = SimpleNamespace(base_url='https://example.test/prod-api/', headers={}, client=None) + query_db = make_query_db() + current_user = make_current_user() + + with ( + patch.object(UploadConfig, 'PRIVATE_UPLOAD_PATH', str(storage_root)), + patch.object(FileInfoDao, 'get_file_info_by_id', new=AsyncMock(return_value=file_info)), + patch.object(CommonService, '_enqueue_file_access_log', new_callable=AsyncMock) as enqueue_audit, + ): + download_result = asyncio.run( + CommonService.download_managed_file_services( + request, + query_db, + current_user, + 'file-id', + range_header='bytes=3-6', + ) + ) + assert asyncio.run(collect_stream(download_result.data)) == b'3456' + + assert download_result.byte_range.start == RANGE_TEST_START + assert download_result.byte_range.end == RANGE_TEST_END + assert download_result.byte_range.length == RANGE_TEST_LENGTH + assert [call.kwargs['result'] for call in enqueue_audit.await_args_list] == ['allowed', 'completed'] + assert enqueue_audit.await_args_list[0].kwargs['operation_detail'] == { + 'rangeStart': RANGE_TEST_START, + 'rangeEnd': RANGE_TEST_END, + 'fileSize': RANGE_TEST_FILE_SIZE, + } + assert enqueue_audit.await_args_list[1].kwargs['bytes_sent'] == RANGE_TEST_LENGTH + + +def test_managed_download_audits_unsatisfied_range_without_disclosing_before_permission(tmp_path: Path) -> None: + storage_root = tmp_path / 'private' + target = storage_root / 'upload' / '2026' / '07' / 'report_20260719120000A001.txt' + target.parent.mkdir(parents=True) + target.write_bytes(b'0123456789') + file_info = SimpleNamespace( + storage_type='local', + access_type='private', + upload_user_id=10, + owner_user_id=10, + expire_time=None, + storage_key='upload/2026/07/report_20260719120000A001.txt', + original_name='report.txt', + ) + request = SimpleNamespace(base_url='https://example.test/prod-api/', headers={}, client=None) + + with ( + patch.object(UploadConfig, 'PRIVATE_UPLOAD_PATH', str(storage_root)), + patch.object(FileInfoDao, 'get_file_info_by_id', new=AsyncMock(return_value=file_info)), + patch.object(CommonService, '_enqueue_file_access_log', new_callable=AsyncMock) as enqueue_audit, + pytest.raises(FileRangeNotSatisfiableException), + ): + asyncio.run( + CommonService.download_managed_file_services( + request, + make_query_db(), + make_current_user(), + 'file-id', + range_header='bytes=20-', + ) + ) + + enqueue_audit.assert_awaited_once() + assert enqueue_audit.await_args.kwargs['result'] == 'failed' + assert enqueue_audit.await_args.kwargs['error_message'] == 'RangeNotSatisfiable' + + +def test_download_headers_force_attachment_and_disable_sniffing() -> None: + byte_range = FileUtil.parse_byte_range('bytes=2-5', 10) + headers = UploadUtil.build_download_headers('../../report.html', byte_range) + + assert headers['Content-Disposition'].startswith('attachment;') + assert headers['download-filename'] == 'report.html' + assert headers['Accept-Ranges'] == 'bytes' + assert headers['Content-Length'] == '4' + assert headers['Content-Range'] == 'bytes 2-5/10' + assert headers['X-Content-Type-Options'] == 'nosniff' + assert headers['Content-Security-Policy'].startswith('sandbox;') + assert headers['X-Frame-Options'] == 'DENY' + + +def test_static_files_force_only_html_to_download(tmp_path: Path) -> None: + html_file = tmp_path / 'legacy.html' + html_file.write_text('', encoding='utf-8') + image_file = tmp_path / 'safe.png' + image_file.write_bytes(b'not-a-real-image') + pdf_file = tmp_path / 'report.pdf' + pdf_file.write_bytes(b'%PDF-1.4') + static_files = SecureStaticFiles(directory=tmp_path) + scope = {'type': 'http', 'method': 'GET', 'path': '/profile/legacy.html', 'headers': []} + + html_response = asyncio.run(static_files.get_response('legacy.html', scope)) + image_response = asyncio.run(static_files.get_response('safe.png', scope)) + pdf_response = asyncio.run(static_files.get_response('report.pdf', scope)) + + assert html_response.headers['Content-Type'] == 'application/octet-stream' + assert html_response.headers['Content-Disposition'].startswith('attachment;') + assert html_response.headers['Content-Security-Policy'] == ( + "sandbox; default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" + ) + assert html_response.headers['X-Content-Type-Options'] == 'nosniff' + assert html_response.headers['X-Frame-Options'] == 'DENY' + assert 'Content-Disposition' not in image_response.headers + assert image_response.headers['X-Content-Type-Options'] == 'nosniff' + assert pdf_response.headers['Content-Type'] == 'application/pdf' + assert 'Content-Disposition' not in pdf_response.headers + assert pdf_response.headers['X-Content-Type-Options'] == 'nosniff' + + +def test_cors_exposes_download_headers() -> None: + app = FastAPI() + add_cors_middleware(app) + + cors_middleware = next(item for item in app.user_middleware if item.cls is CORSMiddleware) + expose_headers = {header.lower() for header in cors_middleware.kwargs['expose_headers']} + + assert 'download-filename' in expose_headers + assert 'content-disposition' in expose_headers + assert 'accept-ranges' in expose_headers + assert 'content-range' in expose_headers + assert 'content-length' in expose_headers diff --git a/ruoyi-fastapi-backend/tests/test_file_access_log.py b/ruoyi-fastapi-backend/tests/test_file_access_log.py new file mode 100644 index 0000000..a3fad78 --- /dev/null +++ b/ruoyi-fastapi-backend/tests/test_file_access_log.py @@ -0,0 +1,165 @@ +import asyncio +import json +import os +import sys +from datetime import datetime +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +from sqlalchemy import Text + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +from config.env import LogConfig +from module_admin.dao.file_access_dao import FileAccessLogDao +from module_admin.entity.do.file_do import SysFileAccessLog +from module_admin.entity.vo.file_vo import FileAccessLogModel +from module_admin.service.file_access_service import FileAuditService +from module_admin.service.log_service import LogAggregatorService, LogQueueService + +FILE_SIZE = 12 +BATCH_FILE_COUNT = 2 +LONG_OPERATION_DETAIL_LENGTH = 3000 + + +class AsyncSessionContext: + """测试用异步数据库会话上下文。""" + + def __init__(self, session: SimpleNamespace) -> None: + self.session = session + + async def __aenter__(self) -> SimpleNamespace: + return self.session + + async def __aexit__(self, exc_type: type | None, exc_value: BaseException | None, traceback: object) -> None: + return None + + +def make_file_access_log() -> FileAccessLogModel: + return FileAccessLogModel( + fileId='file-id', + action='download', + actorUserId=10, + actorName='user10', + result='completed', + requestId='request-id', + traceId='trace-id', + ipAddress='127.0.0.1', + userAgent='pytest', + bytesSent=FILE_SIZE, + operationDetail='{"newStatus":"active"}', + accessTime=datetime(2026, 7, 19, 12, 0, 0), + ) + + +def test_file_access_log_operation_detail_uses_unbounded_text_type() -> None: + operation_detail_column = SysFileAccessLog.__table__.c.operation_detail + + assert isinstance(operation_detail_column.type, Text) + assert operation_detail_column.type.length is None + + +def test_file_access_log_is_enqueued_to_redis_stream() -> None: + redis = SimpleNamespace(xadd=AsyncMock()) + request = SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(redis=redis))) + + with ( + patch('module_admin.service.log_service.TraceCtx.get_request_id', return_value='request-id'), + patch('module_admin.service.log_service.TraceCtx.get_trace_id', return_value='trace-id'), + patch('module_admin.service.log_service.TraceCtx.get_span_id', return_value='span-id'), + ): + asyncio.run(LogQueueService.enqueue_file_access_log(request, make_file_access_log(), 'file:download:completed')) + + redis.xadd.assert_awaited_once() + stream_name, event = redis.xadd.await_args.args + assert stream_name == LogConfig.log_stream_key + assert event['event_type'] == 'file_access' + payload = json.loads(event['payload']) + assert payload['fileId'] == 'file-id' + assert payload['result'] == 'completed' + assert payload['bytesSent'] == FILE_SIZE + assert payload['operationDetail'] == '{"newStatus":"active"}' + + +def test_file_access_log_event_is_persisted_and_acknowledged() -> None: + session = SimpleNamespace(commit=AsyncMock(), rollback=AsyncMock()) + redis = SimpleNamespace(set=AsyncMock(return_value=True), xack=AsyncMock(), delete=AsyncMock()) + file_access_log = make_file_access_log() + messages = [ + ( + '1-0', + { + 'event_type': 'file_access', + 'event_id': 'event-id', + 'payload': json.dumps(file_access_log.model_dump(by_alias=True, exclude_none=True), default=str), + }, + ) + ] + + with ( + patch( + 'module_admin.service.log_service.AsyncSessionLocal', + return_value=AsyncSessionContext(session), + ), + patch.object(FileAccessLogDao, 'add_file_access_log_dao', new_callable=AsyncMock) as add_file_access_log, + ): + asyncio.run(LogAggregatorService._process_messages(redis, LogConfig.log_stream_key, messages)) + + add_file_access_log.assert_awaited_once() + saved_log = add_file_access_log.await_args.args[1] + assert saved_log.file_id == 'file-id' + assert saved_log.result == 'completed' + assert saved_log.operation_detail == '{"newStatus":"active"}' + session.commit.assert_awaited_once() + redis.xack.assert_awaited_once_with(LogConfig.log_stream_key, LogConfig.log_stream_group, '1-0') + + +def test_file_audit_batch_events_use_independent_deduplication_keys() -> None: + redis = SimpleNamespace(xadd=AsyncMock()) + request = SimpleNamespace( + app=SimpleNamespace(state=SimpleNamespace(redis=redis)), + headers={'User-Agent': 'pytest'}, + client=SimpleNamespace(host='127.0.0.1'), + ) + current_user = SimpleNamespace(user=SimpleNamespace(user_id=10, user_name='user10')) + + async def enqueue_batch() -> None: + await FileAuditService.enqueue_file_audit( + request, + current_user, + 'file-id-1', + 'transfer', + 'completed', + operation_detail={'previousOwnerUserId': 10, 'newOwnerUserId': 20}, + ) + await FileAuditService.enqueue_file_audit( + request, + current_user, + 'file-id-2', + 'transfer', + 'completed', + operation_detail={'previousOwnerUserId': 10, 'newOwnerUserId': 20}, + ) + + with ( + patch('module_admin.service.file_access_service.TraceCtx.get_request_id', return_value='request-id'), + patch('module_admin.service.file_access_service.TraceCtx.get_trace_id', return_value='trace-id'), + patch('module_admin.service.log_service.TraceCtx.get_request_id', return_value='request-id'), + patch('module_admin.service.log_service.TraceCtx.get_trace_id', return_value='trace-id'), + patch('module_admin.service.log_service.TraceCtx.get_span_id', return_value='span-id'), + ): + asyncio.run(enqueue_batch()) + + assert redis.xadd.await_count == BATCH_FILE_COUNT + first_event = redis.xadd.await_args_list[0].args[1] + second_event = redis.xadd.await_args_list[1].args[1] + assert first_event['event_id'] != second_event['event_id'] + assert json.loads(first_event['payload'])['operationDetail'] == ('{"previousOwnerUserId":10,"newOwnerUserId":20}') + + +def test_file_audit_operation_detail_is_not_truncated() -> None: + reason = 'a' * LONG_OPERATION_DETAIL_LENGTH + serialized_detail = FileAuditService._serialize_operation_detail({'reason': reason}) + + parsed_detail = json.loads(serialized_detail) + assert parsed_detail['reason'] == reason diff --git a/ruoyi-fastapi-backend/tests/test_file_acl.py b/ruoyi-fastapi-backend/tests/test_file_acl.py new file mode 100644 index 0000000..7c05579 --- /dev/null +++ b/ruoyi-fastapi-backend/tests/test_file_acl.py @@ -0,0 +1,580 @@ +import asyncio +import os +import sys +from datetime import datetime, timedelta +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest +from sqlalchemy import false, true + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +from exceptions.exception import ServiceException +from module_admin.dao.file_access_dao import FileAclDao +from module_admin.dao.file_info_dao import FileInfoDao +from module_admin.entity.vo.file_vo import BatchSaveFileAclModel, SaveFileAclModel +from module_admin.service.common_service import CommonService +from module_admin.service.file_access_service import FileAclService, FileAuditService + +FILE_ID = '11111111-1111-4111-8111-111111111111' +FILE_ID_2 = '22222222-2222-4222-8222-222222222222' +PARENT_DEPT_ID = 100 +CHILD_DEPT_ID = 110 +BATCH_FILE_COUNT = 2 + + +def make_query_db() -> SimpleNamespace: + return SimpleNamespace(commit=AsyncMock(), rollback=AsyncMock()) + + +def make_current_user( + user_id: int, + role_ids: list[int] | None = None, + dept_id: int | None = None, + ancestors: str = '', + admin: bool = False, +) -> SimpleNamespace: + roles = [SimpleNamespace(role_id=role_id) for role_id in (role_ids or [])] + dept = SimpleNamespace(dept_id=dept_id, ancestors=ancestors) if dept_id else None + user = SimpleNamespace( + user_id=user_id, + user_name=f'user{user_id}', + admin=admin, + role=roles, + role_ids=','.join(str(role_id) for role_id in role_ids or []), + dept_id=dept_id, + dept=dept, + ) + return SimpleNamespace(user=user) + + +def make_file_info( + owner_user_id: int = 10, + upload_user_id: int = 11, + uploader_access_enabled: str = '1', +) -> SimpleNamespace: + return SimpleNamespace( + owner_user_id=owner_user_id, + upload_user_id=upload_user_id, + uploader_access_enabled=uploader_access_enabled, + ) + + +def make_acl( + subject_type: str, + subject_id: int, + effect: str = 'allow', + include_children: str = '0', +) -> SimpleNamespace: + return SimpleNamespace( + subject_type=subject_type, + subject_id=subject_id, + effect=effect, + include_children=include_children, + ) + + +@pytest.mark.parametrize( + ('current_user', 'file_acl'), + [ + (make_current_user(20), make_acl('user', 20)), + (make_current_user(20, role_ids=[5]), make_acl('role', 5)), + (make_current_user(20, dept_id=120), make_acl('dept', 120)), + (make_current_user(20, dept_id=120, ancestors='0,100,110'), make_acl('dept', 100, include_children='1')), + ], +) +def test_private_file_acl_allows_matching_user_role_or_department( + current_user: SimpleNamespace, + file_acl: SimpleNamespace, +) -> None: + with patch.object(FileAclDao, 'get_effective_file_acl_list', new=AsyncMock(return_value=[file_acl])): + result = asyncio.run( + CommonService._has_private_file_download_permission( + make_query_db(), + current_user, + make_file_info(), + FILE_ID, + datetime.now(), + ) + ) + + assert result is True + + +def test_private_file_acl_does_not_apply_parent_department_without_include_children() -> None: + current_user = make_current_user(20, dept_id=120, ancestors='0,100,110') + file_acl = make_acl('dept', 100, include_children='0') + + with patch.object(FileAclDao, 'get_effective_file_acl_list', new=AsyncMock(return_value=[file_acl])): + result = asyncio.run( + CommonService._has_private_file_download_permission( + make_query_db(), + current_user, + make_file_info(), + FILE_ID, + datetime.now(), + ) + ) + + assert result is False + + +def test_file_acl_department_options_are_returned_as_tree() -> None: + dept_list = [ + SimpleNamespace(dept_id=PARENT_DEPT_ID, dept_name='研发中心', parent_id=0), + SimpleNamespace(dept_id=CHILD_DEPT_ID, dept_name='平台研发部', parent_id=PARENT_DEPT_ID), + ] + + with patch.object(FileAclDao, 'get_acl_dept_list', new=AsyncMock(return_value=dept_list)): + result = asyncio.run(FileAclService.get_file_acl_dept_tree_services(make_query_db(), true())) + + assert result[0].id == PARENT_DEPT_ID + assert result[0].children[0].id == CHILD_DEPT_ID + + +def test_file_acl_list_returns_current_version() -> None: + acl_version = 7 + with ( + patch.object( + FileInfoDao, + 'get_file_info_detail_by_id', + new=AsyncMock( + return_value=SimpleNamespace( + acl_version=acl_version, + access_type='private', + owner_user_id=10, + upload_user_id=11, + uploader_access_enabled='1', + ) + ), + ), + patch.object(FileAclDao, 'get_file_acl_list', new=AsyncMock(return_value=[])), + patch.object( + FileAclDao, + 'get_acl_subject_name_map', + new=AsyncMock(return_value={('user', 10): '文件所有者', ('user', 11): '文件上传人'}), + ), + ): + result = asyncio.run( + FileAclService.get_file_acl_list_services( + make_query_db(), + FILE_ID, + true(), + true(), + true(), + ) + ) + + assert result.acl_version == acl_version + assert result.entries == [] + assert [item.source for item in result.builtin_permissions] == ['admin', 'owner', 'uploader'] + assert result.builtin_permissions[1].subject_name == '文件所有者' + assert result.builtin_permissions[1].deny_overridable is False + assert result.builtin_permissions[2].subject_name == '文件上传人' + assert result.builtin_permissions[2].enabled is True + assert result.builtin_permissions[2].deny_overridable is True + + +def test_file_acl_list_marks_uploader_permission_non_overridable_when_uploader_is_owner() -> None: + file_info = SimpleNamespace( + acl_version=0, + access_type='private', + owner_user_id=10, + upload_user_id=10, + uploader_access_enabled='1', + ) + with ( + patch.object( + FileInfoDao, + 'get_file_info_detail_by_id', + new=AsyncMock(return_value=file_info), + ), + patch.object(FileAclDao, 'get_file_acl_list', new=AsyncMock(return_value=[])), + patch.object( + FileAclDao, + 'get_acl_subject_name_map', + new=AsyncMock(return_value={('user', 10): '上传人与所有者'}), + ), + ): + result = asyncio.run( + FileAclService.get_file_acl_list_services( + make_query_db(), + FILE_ID, + true(), + true(), + true(), + ) + ) + + uploader_permission = result.builtin_permissions[2] + assert uploader_permission.source == 'uploader' + assert uploader_permission.enabled is True + assert uploader_permission.deny_overridable is False + + +def test_file_acl_list_exposes_disabled_uploader_permission() -> None: + file_info = SimpleNamespace( + access_type='private', + owner_user_id=10, + upload_user_id=11, + uploader_access_enabled='0', + ) + + builtin_permissions = FileAclService._build_builtin_permissions( + file_info, + {('user', 10): '文件所有者', ('user', 11): '文件上传人'}, + ) + + uploader_permission = builtin_permissions[2] + assert uploader_permission.source == 'uploader' + assert uploader_permission.enabled is False + assert uploader_permission.deny_overridable is False + assert uploader_permission.description == '上传人访问权限已在文件转移时移除。' + + +def test_file_acl_role_options_exclude_roles_with_members_outside_user_data_scope() -> None: + query_db = SimpleNamespace( + execute=AsyncMock( + side_effect=[ + SimpleNamespace(all=lambda: [(5, 20), (5, 30), (6, 20)]), + SimpleNamespace(all=lambda: [(5, 20), (6, 20)]), + ] + ) + ) + + result = asyncio.run( + FileAclDao._filter_role_rows_by_data_scope( + query_db, + [(5, '跨部门角色'), (6, '本部门角色')], + true(), + ) + ) + + assert result == [(6, '本部门角色')] + + +def test_private_file_acl_explicit_deny_overrides_uploader_and_allow_rule() -> None: + current_user = make_current_user(20, role_ids=[5]) + file_acl_list = [make_acl('role', 5, effect='allow'), make_acl('user', 20, effect='deny')] + + with patch.object(FileAclDao, 'get_effective_file_acl_list', new=AsyncMock(return_value=file_acl_list)): + result = asyncio.run( + CommonService._has_private_file_download_permission( + make_query_db(), + current_user, + make_file_info(upload_user_id=20), + FILE_ID, + datetime.now(), + ) + ) + + assert result is False + + +def test_private_file_acl_does_not_allow_uploader_when_compatibility_access_is_disabled() -> None: + current_user = make_current_user(20) + + with patch.object(FileAclDao, 'get_effective_file_acl_list', new=AsyncMock(return_value=[])): + result = asyncio.run( + CommonService._has_private_file_download_permission( + make_query_db(), + current_user, + make_file_info(upload_user_id=20, uploader_access_enabled='0'), + FILE_ID, + datetime.now(), + ) + ) + + assert result is False + + +@pytest.mark.parametrize('current_user', [make_current_user(20, admin=True), make_current_user(20)]) +def test_private_file_acl_admin_or_owner_is_not_blocked_by_deny(current_user: SimpleNamespace) -> None: + file_info = make_file_info(owner_user_id=20 if not current_user.user.admin else 10) + get_acl_list = AsyncMock(return_value=[make_acl('user', 20, effect='deny')]) + + with patch.object(FileAclDao, 'get_effective_file_acl_list', new=get_acl_list): + result = asyncio.run( + CommonService._has_private_file_download_permission( + make_query_db(), + current_user, + file_info, + FILE_ID, + datetime.now(), + ) + ) + + assert result is True + get_acl_list.assert_not_awaited() + + +def test_save_file_acl_normalizes_department_scope_and_commits() -> None: + query_db = make_query_db() + current_user = make_current_user(1, admin=True) + acl_version = 3 + file_info = SimpleNamespace(access_type='private', acl_version=acl_version, update_by=None, update_time=None) + save_model = SaveFileAclModel( + aclVersion=acl_version, + entries=[ + { + 'subjectType': 'dept', + 'subjectId': 100, + 'effect': 'allow', + 'includeChildren': True, + 'expireTime': datetime.now() + timedelta(days=1), + }, + { + 'subjectType': 'user', + 'subjectId': 20, + 'effect': 'deny', + 'includeChildren': True, + }, + ], + ) + + with ( + patch.object(FileInfoDao, 'get_file_info_by_id_for_update', new=AsyncMock(return_value=file_info)), + patch.object( + FileAclDao, + 'get_acl_subject_name_map', + new=AsyncMock(return_value={('dept', 100): '研发部门', ('user', 20): '测试用户'}), + ), + patch.object(FileAclDao, 'replace_file_acl_list', new_callable=AsyncMock) as replace_file_acl_list, + patch.object(FileAuditService, 'enqueue_file_audit', new_callable=AsyncMock) as enqueue_file_audit, + ): + result = asyncio.run( + FileAclService.save_file_acl_services( + query_db, + current_user, + FILE_ID, + save_model, + true(), + true(), + true(), + ) + ) + + saved_acl_list = replace_file_acl_list.await_args.args[2] + assert result.is_success is True + assert saved_acl_list[0].include_children == '1' + assert saved_acl_list[1].include_children == '0' + assert file_info.acl_version == acl_version + 1 + assert file_info.update_by == 'user1' + query_db.commit.assert_awaited_once() + enqueue_file_audit.assert_awaited_once() + assert enqueue_file_audit.await_args.args[3:5] == ('acl_update', 'completed') + audit_detail = enqueue_file_audit.await_args.kwargs['operation_detail'] + assert audit_detail['previousAclVersion'] == acl_version + assert audit_detail['newAclVersion'] == acl_version + 1 + assert audit_detail['allowCount'] == 1 + assert audit_detail['denyCount'] == 1 + + +def test_batch_save_file_acl_replaces_all_selected_private_files() -> None: + query_db = make_query_db() + current_user = make_current_user(1, admin=True) + file_infos = [ + SimpleNamespace(file_id=FILE_ID, access_type='private', acl_version=2, update_by=None, update_time=None), + SimpleNamespace(file_id=FILE_ID_2, access_type='private', acl_version=4, update_by=None, update_time=None), + ] + save_model = BatchSaveFileAclModel( + fileIds=f'{FILE_ID},{FILE_ID_2}', + entries=[{'subjectType': 'user', 'subjectId': 20, 'effect': 'allow'}], + ) + + with ( + patch.object(FileInfoDao, 'get_file_infos_by_ids_for_update', new=AsyncMock(return_value=file_infos)), + patch.object( + FileAclDao, + 'get_acl_subject_name_map', + new=AsyncMock(return_value={('user', 20): '测试用户'}), + ), + patch.object(FileAclDao, 'replace_file_acl_lists', new_callable=AsyncMock) as replace_file_acl_lists, + patch.object(FileAuditService, 'enqueue_file_audit', new_callable=AsyncMock) as enqueue_file_audit, + ): + result = asyncio.run( + FileAclService.batch_save_file_acl_services( + query_db, + current_user, + save_model, + true(), + true(), + true(), + ) + ) + + saved_acl_list = replace_file_acl_lists.await_args.args[2] + assert result.is_success is True + assert {item.file_id for item in saved_acl_list} == {FILE_ID, FILE_ID_2} + assert [file_info.acl_version for file_info in file_infos] == [3, 5] + assert enqueue_file_audit.await_count == BATCH_FILE_COUNT + assert all(item.kwargs['operation_detail']['batch'] is True for item in enqueue_file_audit.await_args_list) + query_db.commit.assert_awaited_once() + + +def test_batch_save_file_acl_rejects_public_files() -> None: + query_db = make_query_db() + save_model = BatchSaveFileAclModel(fileIds=FILE_ID, entries=[]) + file_infos = [SimpleNamespace(file_id=FILE_ID, access_type='public', acl_version=0)] + + with ( + patch.object(FileInfoDao, 'get_file_infos_by_ids_for_update', new=AsyncMock(return_value=file_infos)), + patch.object(FileAclDao, 'replace_file_acl_lists', new_callable=AsyncMock) as replace_file_acl_lists, + pytest.raises(ServiceException) as public_error, + ): + asyncio.run( + FileAclService.batch_save_file_acl_services( + query_db, + make_current_user(1, admin=True), + save_model, + true(), + true(), + true(), + ) + ) + + assert public_error.value.message == '批量授权仅支持受保护文件' + replace_file_acl_lists.assert_not_awaited() + query_db.rollback.assert_awaited_once() + + +def test_save_file_acl_rejects_duplicate_or_expired_entries() -> None: + query_db = make_query_db() + current_user = make_current_user(1, admin=True) + file_info = SimpleNamespace(access_type='private', acl_version=0) + duplicate_model = SaveFileAclModel( + aclVersion=0, + entries=[ + {'subjectType': 'user', 'subjectId': 20, 'effect': 'allow'}, + {'subjectType': 'user', 'subjectId': 20, 'effect': 'deny'}, + ], + ) + expired_model = SaveFileAclModel( + aclVersion=0, + entries=[ + { + 'subjectType': 'user', + 'subjectId': 20, + 'effect': 'allow', + 'expireTime': datetime.now() - timedelta(seconds=1), + } + ], + ) + + with patch.object(FileInfoDao, 'get_file_info_by_id_for_update', new=AsyncMock(return_value=file_info)): + with pytest.raises(ServiceException) as duplicate_error: + asyncio.run( + FileAclService.save_file_acl_services( + query_db, + current_user, + FILE_ID, + duplicate_model, + true(), + true(), + true(), + ) + ) + with pytest.raises(ServiceException) as expired_error: + asyncio.run( + FileAclService.save_file_acl_services( + query_db, + current_user, + FILE_ID, + expired_model, + true(), + true(), + true(), + ) + ) + + assert duplicate_error.value.message == '同一授权主体不能重复配置' + assert expired_error.value.message == '授权过期时间必须晚于当前时间' + + +def test_save_file_acl_rejects_subject_outside_data_scope() -> None: + query_db = make_query_db() + current_user = make_current_user(1, admin=True) + file_info = SimpleNamespace(access_type='private', acl_version=0) + save_model = SaveFileAclModel( + aclVersion=0, + entries=[{'subjectType': 'user', 'subjectId': 20, 'effect': 'allow'}], + ) + + with ( + patch.object(FileInfoDao, 'get_file_info_by_id_for_update', new=AsyncMock(return_value=file_info)), + patch.object(FileAclDao, 'get_acl_subject_name_map', new=AsyncMock(return_value={})), + pytest.raises(ServiceException) as scope_error, + ): + asyncio.run( + FileAclService.save_file_acl_services( + query_db, + current_user, + FILE_ID, + save_model, + true(), + true(), + true(), + ) + ) + + assert scope_error.value.message == '部分授权主体不存在、已停用或超出数据权限' + + +def test_save_file_acl_rejects_file_outside_data_scope() -> None: + query_db = make_query_db() + current_user = make_current_user(20) + save_model = SaveFileAclModel(aclVersion=0, entries=[]) + file_data_scope_sql = false() + + with ( + patch.object( + FileInfoDao, + 'get_file_info_by_id_for_update', + new=AsyncMock(return_value=None), + ) as get_file_info, + patch.object(FileAclDao, 'replace_file_acl_list', new_callable=AsyncMock) as replace_file_acl_list, + pytest.raises(ServiceException) as scope_error, + ): + asyncio.run( + FileAclService.save_file_acl_services( + query_db, + current_user, + FILE_ID, + save_model, + file_data_scope_sql, + true(), + true(), + ) + ) + + assert get_file_info.await_args.args[2] is file_data_scope_sql + assert scope_error.value.message == '文件信息不存在、已删除或超出数据权限' + replace_file_acl_list.assert_not_awaited() + + +def test_save_file_acl_rejects_stale_acl_version() -> None: + query_db = make_query_db() + file_info = SimpleNamespace(access_type='private', acl_version=2) + save_model = SaveFileAclModel(aclVersion=1, entries=[]) + + with ( + patch.object(FileInfoDao, 'get_file_info_by_id_for_update', new=AsyncMock(return_value=file_info)), + patch.object(FileAclDao, 'replace_file_acl_list', new_callable=AsyncMock) as replace_file_acl_list, + pytest.raises(ServiceException) as version_error, + ): + asyncio.run( + FileAclService.save_file_acl_services( + query_db, + make_current_user(1, admin=True), + FILE_ID, + save_model, + true(), + true(), + true(), + ) + ) + + assert version_error.value.message == '文件权限已被其他用户修改，请刷新后重试' + replace_file_acl_list.assert_not_awaited() + query_db.rollback.assert_awaited_once() diff --git a/ruoyi-fastapi-backend/tests/test_file_lifecycle_retention_execution.py b/ruoyi-fastapi-backend/tests/test_file_lifecycle_retention_execution.py new file mode 100644 index 0000000..5790614 --- /dev/null +++ b/ruoyi-fastapi-backend/tests/test_file_lifecycle_retention_execution.py @@ -0,0 +1,402 @@ +import asyncio +import os +import sys +from datetime import datetime, timedelta +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest +from sqlalchemy import true + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +from config.env import UploadConfig +from exceptions.exception import ServiceException +from module_admin.dao.file_business_dao import FileReferenceDao, FileRetentionNoticeDao +from module_admin.dao.file_info_dao import FileInfoDao +from module_admin.entity.vo.file_vo import ( + DisposeExpiredFileModel, + ExtendFileRetentionModel, + FileRetentionNoticePageQueryModel, +) +from module_admin.service.file_access_service import FileAuditService +from module_admin.service.file_business_service import ( + FileReferenceService, + FileRetentionNoticeService, + FileRetentionPolicyService, +) +from module_admin.service.file_service import FileLifecycleService, FileRetentionDispositionService +from utils.file_util import FileUtil + +FILE_ID = '11111111-1111-4111-8111-111111111111' +PURGE_COMMIT_COUNT = 2 + + +def make_query_db() -> SimpleNamespace: + return SimpleNamespace(commit=AsyncMock(), rollback=AsyncMock()) + + +def make_current_user() -> SimpleNamespace: + return SimpleNamespace(user=SimpleNamespace(user_id=1, user_name='admin', admin=True)) + + +def make_file_info( + access_type: str = 'private', + expire_time: datetime | None = None, +) -> SimpleNamespace: + return SimpleNamespace( + file_id=FILE_ID, + original_name='private.txt', + storage_type='local', + access_type=access_type, + storage_key='upload/private.txt', + stored_name='private.txt', + expire_time=expire_time, + business_type=None, + business_id=None, + update_by='', + update_time=None, + ) + + +def test_purge_file_removes_recycle_content_and_metadata(tmp_path: Path) -> None: + public_root = tmp_path / 'public' + trash_root = tmp_path / 'trash' + trash_file = trash_root / FILE_ID / 'private.txt' + trash_file.parent.mkdir(parents=True) + trash_file.write_bytes(b'content') + file_info = make_file_info(access_type='public') + query_db = make_query_db() + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(public_root)), + patch.object(UploadConfig, 'FILE_TRASH_PATH', str(trash_root)), + patch.object( + FileInfoDao, + 'get_purgeable_file_infos_by_ids_for_update', + new=AsyncMock(return_value=[file_info]), + ), + patch.object( + FileReferenceService, + 'get_file_reference_count_map_services', + new=AsyncMock(return_value={}), + ), + patch.object(FileInfoDao, 'mark_file_infos_purging', new_callable=AsyncMock) as mark_purging, + patch.object(FileInfoDao, 'purge_file_infos', new_callable=AsyncMock) as purge_file_infos, + patch.object(FileAuditService, 'enqueue_file_audit', new_callable=AsyncMock) as enqueue_file_audit, + ): + result = asyncio.run( + FileLifecycleService.purge_file_services( + query_db, + make_current_user(), + FILE_ID, + true(), + ) + ) + + assert result.is_success is True + assert not trash_file.exists() + mark_purging.assert_awaited_once() + purge_file_infos.assert_awaited_once_with(query_db, [FILE_ID]) + assert query_db.commit.await_count == PURGE_COMMIT_COUNT + assert enqueue_file_audit.await_args.args[2:5] == (FILE_ID, 'purge', 'completed') + + +def test_purge_file_rejects_business_reference(tmp_path: Path) -> None: + file_info = make_file_info() + query_db = make_query_db() + + with ( + patch.object( + FileInfoDao, + 'get_purgeable_file_infos_by_ids_for_update', + new=AsyncMock(return_value=[file_info]), + ), + patch.object( + FileReferenceService, + 'get_file_reference_count_map_services', + new=AsyncMock(return_value={FILE_ID: 1}), + ), + patch.object(FileInfoDao, 'mark_file_infos_purging', new_callable=AsyncMock) as mark_purging, + pytest.raises(ServiceException) as reference_error, + ): + asyncio.run( + FileLifecycleService.purge_file_services( + query_db, + make_current_user(), + FILE_ID, + true(), + ) + ) + + assert reference_error.value.message == '部分文件仍被业务引用，不能永久清理' + mark_purging.assert_not_awaited() + query_db.rollback.assert_awaited_once() + + +def test_retention_scan_creates_expiring_and_expired_notices() -> None: + current_time = datetime.now() + expired_file = make_file_info(expire_time=current_time - timedelta(days=1)) + expiring_file = SimpleNamespace( + **{ + **expired_file.__dict__, + 'file_id': '22222222-2222-4222-8222-222222222222', + 'expire_time': current_time + timedelta(days=1), + } + ) + query_db = make_query_db() + + with ( + patch.object( + FileRetentionNoticeDao, + 'get_missing_notice_candidates', + new=AsyncMock(side_effect=[[expired_file], [expiring_file]]), + ), + patch.object( + FileRetentionNoticeDao, + 'invalidate_expiring_notices', + new_callable=AsyncMock, + ) as invalidate_notices, + patch.object( + FileRetentionNoticeDao, + 'add_file_retention_notices', + new_callable=AsyncMock, + ) as add_notices, + ): + result = asyncio.run( + FileRetentionNoticeService.scan_file_retention_notices_services( + query_db, + remind_days=7, + batch_size=100, + file_data_scope_sql=true(), + ) + ) + + assert result.expiring_count == 1 + assert result.expired_count == 1 + invalidate_notices.assert_awaited_once_with(query_db, [FILE_ID]) + notice_list = add_notices.await_args.args[1] + assert [(notice.file_id, notice.notice_type) for notice in notice_list] == [ + (FILE_ID, 'expired'), + (expiring_file.file_id, 'expiring'), + ] + query_db.commit.assert_awaited_once() + + +def test_mark_retention_notice_read_checks_data_scope() -> None: + query_db = make_query_db() + query_object = FileRetentionNoticePageQueryModel() + + with ( + patch.object( + FileRetentionNoticeDao, + 'get_file_retention_notice_list', + new=AsyncMock(return_value=[]), + ) as get_notice_list, + patch.object( + FileRetentionNoticeDao, + 'get_notice_ids_in_data_scope_for_update', + new=AsyncMock(return_value=[1, 2]), + ), + patch.object( + FileRetentionNoticeDao, + 'mark_file_retention_notices_read', + new_callable=AsyncMock, + ) as mark_notices_read, + ): + result = asyncio.run( + FileRetentionNoticeService.get_file_retention_notice_list_services( + query_db, + query_object, + true(), + is_page=True, + ) + ) + read_result = asyncio.run( + FileRetentionNoticeService.mark_file_retention_notices_read_services( + query_db, + '1,2', + 'admin', + true(), + ) + ) + + assert result == [] + assert get_notice_list.await_args.args[3] is True + assert read_result.is_success is True + assert mark_notices_read.await_args.args[1] == [1, 2] + query_db.commit.assert_awaited_once() + + +def test_retention_policy_rejects_public_business_file() -> None: + query_db = make_query_db() + public_file = make_file_info(access_type='public') + + with ( + patch.object( + FileInfoDao, + 'get_file_infos_by_ids_for_update', + new=AsyncMock(return_value=[public_file]), + ), + patch.object( + FileRetentionPolicyService, + 'get_enabled_file_retention_policy_services', + new=AsyncMock(return_value=SimpleNamespace(retention_days=30)), + ), + patch.object( + FileReferenceDao, + 'replace_business_file_references', + new_callable=AsyncMock, + ) as replace_references, + pytest.raises(ServiceException) as access_type_error, + ): + asyncio.run( + FileReferenceService.replace_business_file_references_services( + query_db, + 'notice', + '1', + [FILE_ID], + 'admin', + true(), + ) + ) + + assert access_type_error.value.message == '配置保留策略的业务只能引用受保护文件' + replace_references.assert_not_awaited() + + +def test_extend_file_retention_updates_terminal_references() -> None: + current_time = datetime.now() + previous_expire_time = current_time + timedelta(days=1) + new_expire_time = current_time + timedelta(days=31) + file_info = make_file_info(expire_time=previous_expire_time) + reference = SimpleNamespace( + reference_id=1, + retention_expire_time=previous_expire_time, + ) + query_db = make_query_db() + + with ( + patch.object( + FileRetentionNoticeDao, + 'get_file_retention_notice_context_for_update', + new=AsyncMock(return_value=(SimpleNamespace(notice_id=1), file_info)), + ), + patch.object( + FileReferenceDao, + 'get_file_reference_list_for_update', + new=AsyncMock(return_value=[reference]), + ), + patch.object( + FileRetentionNoticeDao, + 'invalidate_file_retention_notices', + new_callable=AsyncMock, + ) as invalidate_notices, + patch.object(FileAuditService, 'enqueue_file_audit', new_callable=AsyncMock) as enqueue_file_audit, + ): + result = asyncio.run( + FileRetentionDispositionService.extend_file_retention_services( + query_db, + make_current_user(), + 1, + ExtendFileRetentionModel(expireTime=new_expire_time, reason='业务继续留存'), + true(), + request=SimpleNamespace(), + ) + ) + + assert result.is_success is True + assert file_info.expire_time == new_expire_time + assert reference.retention_expire_time == new_expire_time + invalidate_notices.assert_awaited_once_with(query_db, FILE_ID) + query_db.commit.assert_awaited_once() + assert enqueue_file_audit.await_args.args[3:5] == ('retention_extend', 'completed') + + +def test_dispose_expired_file_releases_expired_references() -> None: + current_time = datetime.now() + expire_time = current_time - timedelta(days=1) + file_info = make_file_info(expire_time=expire_time) + reference = SimpleNamespace( + reference_id=1, + business_type='notice', + business_id='1', + business_name='测试公告', + retention_expire_time=expire_time, + ) + query_db = make_query_db() + + with ( + patch.object( + FileRetentionNoticeDao, + 'get_file_retention_notice_context_for_update', + new=AsyncMock(return_value=(SimpleNamespace(notice_id=1), file_info)), + ), + patch.object( + FileReferenceDao, + 'get_file_reference_list_for_update', + new=AsyncMock(return_value=[reference]), + ), + patch.object(FileUtil, 'stage_file_deletions', return_value=[]), + patch.object(FileReferenceDao, 'delete_file_references', new_callable=AsyncMock) as delete_references, + patch.object(FileInfoDao, 'soft_delete_file_infos', new_callable=AsyncMock) as soft_delete_file_infos, + patch.object(FileAuditService, 'enqueue_file_audit', new_callable=AsyncMock) as enqueue_file_audit, + ): + result = asyncio.run( + FileRetentionDispositionService.dispose_expired_file_services( + query_db, + make_current_user(), + 1, + DisposeExpiredFileModel(reason='保留期已结束'), + true(), + request=SimpleNamespace(), + ) + ) + + assert result.is_success is True + delete_references.assert_awaited_once_with(query_db, FILE_ID) + soft_delete_file_infos.assert_awaited_once() + query_db.commit.assert_awaited_once() + assert enqueue_file_audit.await_args.args[3:5] == ('retention_dispose', 'completed') + assert enqueue_file_audit.await_args.kwargs['operation_detail']['releasedReferenceCount'] == 1 + + +def test_dispose_expired_file_rejects_active_reference() -> None: + expire_time = datetime.now() - timedelta(days=1) + file_info = make_file_info(expire_time=expire_time) + reference = SimpleNamespace( + reference_id=1, + retention_expire_time=None, + ) + query_db = make_query_db() + + with ( + patch.object( + FileRetentionNoticeDao, + 'get_file_retention_notice_context_for_update', + new=AsyncMock(return_value=(SimpleNamespace(notice_id=1), file_info)), + ), + patch.object( + FileReferenceDao, + 'get_file_reference_list_for_update', + new=AsyncMock(return_value=[reference]), + ), + patch.object(FileUtil, 'stage_file_deletions') as stage_file_deletions, + patch.object(FileReferenceDao, 'delete_file_references', new_callable=AsyncMock) as delete_references, + pytest.raises(ServiceException) as active_reference_error, + ): + asyncio.run( + FileRetentionDispositionService.dispose_expired_file_services( + query_db, + make_current_user(), + 1, + DisposeExpiredFileModel(reason='保留期已结束'), + true(), + ) + ) + + assert active_reference_error.value.message == '文件存在永久或尚未到期的业务引用，不能执行到期处置' + stage_file_deletions.assert_not_called() + delete_references.assert_not_awaited() + query_db.rollback.assert_awaited_once() diff --git a/ruoyi-fastapi-backend/tests/test_file_management.py b/ruoyi-fastapi-backend/tests/test_file_management.py new file mode 100644 index 0000000..a2cf1d0 --- /dev/null +++ b/ruoyi-fastapi-backend/tests/test_file_management.py @@ -0,0 +1,876 @@ +import asyncio +import os +import sys +from collections.abc import Generator +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest +from sqlalchemy import false, true + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +from common.aspect.data_scope import GetDataScope +from config.env import UploadConfig +from exceptions.exception import ServiceException +from module_admin.dao.file_business_dao import FileReferenceDao +from module_admin.dao.file_info_dao import FileInfoDao +from module_admin.entity.do.file_do import SysFileInfo +from module_admin.entity.vo.file_vo import DeleteFileModel, TransferFileModel +from module_admin.service.file_access_service import FileAuditService +from module_admin.service.file_service import FileLifecycleService, FileQueryService, FileTransferService +from utils.file_util import FileUtil +from utils.upload_util import UploadUtil + +PUBLIC_FILE_ID = '11111111-1111-4111-8111-111111111111' +PRIVATE_FILE_ID = '22222222-2222-4222-8222-222222222222' +TARGET_USER_ID = 20 +TARGET_DEPT_ID = 110 +BATCH_FILE_COUNT = 2 + + +def make_query_db(commit_error: Exception | None = None) -> SimpleNamespace: + commit = AsyncMock(side_effect=commit_error) if commit_error else AsyncMock() + return SimpleNamespace(commit=commit, rollback=AsyncMock()) + + +def make_current_user() -> SimpleNamespace: + return SimpleNamespace(user=SimpleNamespace(user_id=1, user_name='admin', admin=True)) + + +def make_file_info( + file_id: str, + access_type: str, + storage_key: str, + stored_name: str, + business_type: str | None = None, + business_id: str | None = None, +) -> SimpleNamespace: + return SimpleNamespace( + file_id=file_id, + original_name=stored_name, + storage_type='local', + access_type=access_type, + storage_key=storage_key, + stored_name=stored_name, + business_type=business_type, + business_id=business_id, + ) + + +def expire_model_attributes(*models: SimpleNamespace) -> None: + """模拟SQLAlchemy事务结束后ORM属性过期。""" + for model in models: + model.__dict__.clear() + + +@pytest.fixture(autouse=True) +def mock_file_reference_count() -> Generator[None, None, None]: + """默认模拟文件不存在业务引用。""" + with patch.object(FileReferenceDao, 'get_file_reference_count_map', new=AsyncMock(return_value={})): + yield + + +@pytest.mark.parametrize('data_scope', [GetDataScope.DATA_SCOPE_DEPT, GetDataScope.DATA_SCOPE_DEPT_AND_CHILD]) +def test_file_data_scope_does_not_match_unowned_files_when_user_has_no_department(data_scope: str) -> None: + current_user = SimpleNamespace( + user=SimpleNamespace( + user_id=20, + dept_id=None, + admin=False, + role=[SimpleNamespace(role_id=2, data_scope=data_scope)], + ) + ) + + with ( + patch('common.aspect.data_scope.DependencyUtil.check_exclude_routes'), + patch('common.aspect.data_scope.RequestContext.get_current_user', return_value=current_user), + ): + file_data_scope_sql = GetDataScope( + SysFileInfo, + user_alias='owner_user_id', + dept_alias='dept_id', + )(SimpleNamespace()) + + assert str(file_data_scope_sql).lower() == 'false' + + +def test_file_detail_reads_sqlalchemy_attributes_and_outputs_camel_case() -> None: + file_info = SysFileInfo( + file_id=PUBLIC_FILE_ID, + original_name='report.pdf', + stored_name='report_20260720120000A001.pdf', + storage_key='upload/2026/07/20/report_20260720120000A001.pdf', + storage_type='local', + access_type='public', + acl_version=0, + extension='pdf', + file_size=7, + file_hash='a' * 64, + status='active', + del_flag='0', + ) + + file_info_dict = {key: value for key, value in file_info.__dict__.items() if key != '_sa_instance_state'} + with patch.object( + FileInfoDao, + 'get_file_management_detail_by_id', + new=AsyncMock(return_value=file_info_dict), + ): + result = asyncio.run(FileQueryService.file_detail_services(make_query_db(), PUBLIC_FILE_ID, true())) + + assert result.file_id == PUBLIC_FILE_ID + assert result.model_dump(by_alias=True)['originalName'] == 'report.pdf' + assert result.model_dump(by_alias=True)['uploaderAccessEnabled'] == '1' + + +def test_file_storage_status_distinguishes_normal_quarantined_and_missing(tmp_path: Path) -> None: + public_root = tmp_path / 'public' + trash_root = tmp_path / 'trash' + source_file = public_root / 'upload' / 'public.txt' + trash_file = trash_root / PUBLIC_FILE_ID / 'public.txt' + source_file.parent.mkdir(parents=True) + trash_file.parent.mkdir(parents=True) + source_file.write_bytes(b'content') + file_info = { + 'fileId': PUBLIC_FILE_ID, + 'storageType': 'local', + 'accessType': 'public', + 'storageKey': 'upload/public.txt', + 'storedName': 'public.txt', + 'status': 'active', + } + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(public_root)), + patch.object(UploadConfig, 'FILE_TRASH_PATH', str(trash_root)), + ): + assert FileUtil.get_storage_status(file_info) == 'normal' + source_file.replace(trash_file) + file_info['status'] = 'deleted' + assert FileUtil.get_storage_status(file_info) == 'quarantined' + trash_file.unlink() + assert FileUtil.get_storage_status(file_info) == 'missing' + + +def test_file_storage_status_marks_invalid_paths() -> None: + file_info = { + 'fileId': PUBLIC_FILE_ID, + 'storageType': 'local', + 'accessType': 'public', + 'storageKey': '../outside.txt', + 'storedName': 'outside.txt', + 'status': 'active', + } + + assert FileUtil.get_storage_status(file_info) == 'invalid' + + +def test_transfer_file_updates_owner_and_department_with_data_scope() -> None: + query_db = make_query_db() + transfer_model = TransferFileModel( + ownerUserId=TARGET_USER_ID, + deptId=TARGET_DEPT_ID, + retainUploaderAccess=False, + reason='岗位调整', + ) + target_user = SimpleNamespace(user_id=TARGET_USER_ID, user_name='target-user', dept_id=TARGET_DEPT_ID) + target_dept = SimpleNamespace(dept_id=TARGET_DEPT_ID) + source_files = [ + SimpleNamespace( + file_id=PUBLIC_FILE_ID, + owner_user_id=10, + dept_id=100, + uploader_access_enabled='1', + ) + ] + query_db.commit.side_effect = lambda: expire_model_attributes(target_user, target_dept, *source_files) + + with ( + patch.object(FileInfoDao, 'get_transfer_user_by_id', new=AsyncMock(return_value=target_user)) as get_user, + patch.object(FileInfoDao, 'get_transfer_dept_by_id', new=AsyncMock(return_value=target_dept)) as get_dept, + patch.object( + FileInfoDao, + 'get_file_infos_by_ids_for_update', + new=AsyncMock(return_value=source_files), + ) as get_files, + patch.object(FileInfoDao, 'transfer_file_infos', new_callable=AsyncMock) as transfer_files, + patch.object(FileAuditService, 'enqueue_file_audit', new_callable=AsyncMock) as enqueue_file_audit, + ): + result = asyncio.run( + FileTransferService.transfer_file_services( + query_db, + make_current_user(), + PUBLIC_FILE_ID, + transfer_model, + true(), + true(), + true(), + ) + ) + + assert result.is_success is True + assert get_user.await_args.args[1] == TARGET_USER_ID + assert get_dept.await_args.args[1] == TARGET_DEPT_ID + assert get_files.await_args.args[2].compare(true()) + assert transfer_files.await_args.args[1:5] == ([PUBLIC_FILE_ID], TARGET_USER_ID, TARGET_DEPT_ID, False) + query_db.commit.assert_awaited_once() + enqueue_file_audit.assert_awaited_once() + assert enqueue_file_audit.await_args.args[2:5] == (PUBLIC_FILE_ID, 'transfer', 'completed') + assert enqueue_file_audit.await_args.kwargs['operation_detail']['reason'] == '岗位调整' + assert enqueue_file_audit.await_args.kwargs['operation_detail']['previousUploaderAccessEnabled'] is True + assert enqueue_file_audit.await_args.kwargs['operation_detail']['newUploaderAccessEnabled'] is False + + +def test_transfer_file_model_retains_uploader_access_by_default() -> None: + transfer_model = TransferFileModel(ownerUserId=TARGET_USER_ID, deptId=TARGET_DEPT_ID, reason='岗位调整') + + assert transfer_model.retain_uploader_access is True + + +def test_transfer_file_rejects_target_outside_data_scope() -> None: + query_db = make_query_db() + transfer_model = TransferFileModel(ownerUserId=TARGET_USER_ID, deptId=TARGET_DEPT_ID, reason='岗位调整') + user_data_scope_sql = false() + + with ( + patch.object(FileInfoDao, 'get_transfer_user_by_id', new=AsyncMock(return_value=None)) as get_user, + patch.object( + FileInfoDao, + 'get_transfer_dept_by_id', + new=AsyncMock(return_value=SimpleNamespace(dept_id=TARGET_DEPT_ID)), + ), + patch.object(FileInfoDao, 'transfer_file_infos', new_callable=AsyncMock) as transfer_files, + pytest.raises(ServiceException) as scope_error, + ): + asyncio.run( + FileTransferService.transfer_file_services( + query_db, + make_current_user(), + PUBLIC_FILE_ID, + transfer_model, + true(), + user_data_scope_sql, + true(), + ) + ) + + assert get_user.await_args.args[2] is user_data_scope_sql + assert scope_error.value.message == '目标用户或部门不存在、已停用或超出数据权限' + transfer_files.assert_not_awaited() + query_db.rollback.assert_awaited_once() + + +def test_transfer_file_rejects_owner_department_mismatch() -> None: + query_db = make_query_db() + transfer_model = TransferFileModel(ownerUserId=TARGET_USER_ID, deptId=TARGET_DEPT_ID, reason='岗位调整') + + with ( + patch.object( + FileInfoDao, + 'get_transfer_user_by_id', + new=AsyncMock(return_value=SimpleNamespace(user_id=TARGET_USER_ID, dept_id=120)), + ), + patch.object( + FileInfoDao, + 'get_transfer_dept_by_id', + new=AsyncMock(return_value=SimpleNamespace(dept_id=TARGET_DEPT_ID)), + ), + patch.object(FileInfoDao, 'transfer_file_infos', new_callable=AsyncMock) as transfer_files, + pytest.raises(ServiceException) as mismatch_error, + ): + asyncio.run( + FileTransferService.transfer_file_services( + query_db, + make_current_user(), + PUBLIC_FILE_ID, + transfer_model, + true(), + true(), + true(), + ) + ) + + assert mismatch_error.value.message == '目标用户不属于所选部门' + transfer_files.assert_not_awaited() + query_db.rollback.assert_awaited_once() + + +def test_transfer_file_rejects_source_outside_data_scope() -> None: + query_db = make_query_db() + transfer_model = TransferFileModel(ownerUserId=TARGET_USER_ID, deptId=TARGET_DEPT_ID, reason='岗位调整') + file_data_scope_sql = false() + + with ( + patch.object( + FileInfoDao, + 'get_transfer_user_by_id', + new=AsyncMock(return_value=SimpleNamespace(user_id=TARGET_USER_ID, dept_id=TARGET_DEPT_ID)), + ), + patch.object( + FileInfoDao, + 'get_transfer_dept_by_id', + new=AsyncMock(return_value=SimpleNamespace(dept_id=TARGET_DEPT_ID)), + ), + patch.object(FileInfoDao, 'get_file_infos_by_ids_for_update', new=AsyncMock(return_value=[])) as get_files, + patch.object(FileInfoDao, 'transfer_file_infos', new_callable=AsyncMock) as transfer_files, + pytest.raises(ServiceException) as scope_error, + ): + asyncio.run( + FileTransferService.transfer_file_services( + query_db, + make_current_user(), + PUBLIC_FILE_ID, + transfer_model, + file_data_scope_sql, + true(), + true(), + ) + ) + + assert get_files.await_args.args[2] is file_data_scope_sql + assert scope_error.value.message == '部分文件不存在、已删除或超出数据权限' + transfer_files.assert_not_awaited() + query_db.rollback.assert_awaited_once() + + +def test_delete_file_moves_both_storage_types_to_recycle_bin(tmp_path: Path) -> None: + public_root = tmp_path / 'public' + private_root = tmp_path / 'private' + trash_root = tmp_path / 'trash' + public_file = public_root / 'upload' / 'public.txt' + private_file = private_root / 'upload' / 'private.txt' + public_file.parent.mkdir(parents=True) + private_file.parent.mkdir(parents=True) + public_file.write_bytes(b'public-content') + private_file.write_bytes(b'private-content') + file_infos = [ + make_file_info(PUBLIC_FILE_ID, 'public', 'upload/public.txt', 'public.txt'), + make_file_info(PRIVATE_FILE_ID, 'private', 'upload/private.txt', 'private.txt'), + ] + query_db = make_query_db() + query_db.commit.side_effect = lambda: expire_model_attributes(*file_infos) + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(public_root)), + patch.object(UploadConfig, 'PRIVATE_UPLOAD_PATH', str(private_root)), + patch.object(UploadConfig, 'FILE_TRASH_PATH', str(trash_root)), + patch.object(FileInfoDao, 'get_file_infos_by_ids_for_update', new=AsyncMock(return_value=file_infos)), + patch.object(FileInfoDao, 'soft_delete_file_infos', new_callable=AsyncMock) as soft_delete_file_infos, + patch.object(FileAuditService, 'enqueue_file_audit', new_callable=AsyncMock) as enqueue_file_audit, + ): + result = asyncio.run( + FileLifecycleService.delete_file_services( + query_db, + make_current_user(), + DeleteFileModel(fileIds=f'{PUBLIC_FILE_ID},{PRIVATE_FILE_ID}'), + true(), + ) + ) + + assert result.is_success is True + assert not public_file.exists() + assert not private_file.exists() + assert (trash_root / PUBLIC_FILE_ID / 'public.txt').read_bytes() == b'public-content' + assert (trash_root / PRIVATE_FILE_ID / 'private.txt').read_bytes() == b'private-content' + assert soft_delete_file_infos.await_args.args[1] == [PUBLIC_FILE_ID, PRIVATE_FILE_ID] + query_db.commit.assert_awaited_once() + assert enqueue_file_audit.await_count == BATCH_FILE_COUNT + assert [item.args[2:5] for item in enqueue_file_audit.await_args_list] == [ + (PUBLIC_FILE_ID, 'delete', 'completed'), + (PRIVATE_FILE_ID, 'delete', 'completed'), + ] + + +def test_delete_file_rejects_business_reference_before_moving_file(tmp_path: Path) -> None: + public_root = tmp_path / 'public' + public_file = public_root / 'upload' / 'public.txt' + public_file.parent.mkdir(parents=True) + public_file.write_bytes(b'public-content') + file_infos = [make_file_info(PUBLIC_FILE_ID, 'public', 'upload/public.txt', 'public.txt')] + query_db = make_query_db() + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(public_root)), + patch.object(FileInfoDao, 'get_file_infos_by_ids_for_update', new=AsyncMock(return_value=file_infos)), + patch.object( + FileReferenceDao, + 'get_file_reference_count_map', + new=AsyncMock(return_value={PUBLIC_FILE_ID: 1}), + ), + patch.object(FileInfoDao, 'soft_delete_file_infos', new_callable=AsyncMock) as soft_delete_file_infos, + patch.object(FileAuditService, 'enqueue_file_audit', new_callable=AsyncMock) as enqueue_file_audit, + pytest.raises(ServiceException) as reference_error, + ): + asyncio.run( + FileLifecycleService.delete_file_services( + query_db, + make_current_user(), + DeleteFileModel(fileIds=PUBLIC_FILE_ID), + true(), + ) + ) + + assert reference_error.value.message == '文件“public.txt”仍被业务引用，请先解除引用后再删除' + assert public_file.read_bytes() == b'public-content' + soft_delete_file_infos.assert_not_awaited() + query_db.rollback.assert_awaited_once() + assert enqueue_file_audit.await_args.args[2:5] == (PUBLIC_FILE_ID, 'delete', 'denied') + assert enqueue_file_audit.await_args.kwargs['operation_detail']['referenceCount'] == 1 + + +def test_delete_file_rejects_legacy_business_reference(tmp_path: Path) -> None: + public_root = tmp_path / 'public' + public_file = public_root / 'upload' / 'public.txt' + public_file.parent.mkdir(parents=True) + public_file.write_bytes(b'public-content') + file_infos = [ + make_file_info( + PUBLIC_FILE_ID, + 'public', + 'upload/public.txt', + 'public.txt', + business_type='notice', + business_id='1', + ) + ] + query_db = make_query_db() + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(public_root)), + patch.object(FileInfoDao, 'get_file_infos_by_ids_for_update', new=AsyncMock(return_value=file_infos)), + patch.object(FileInfoDao, 'soft_delete_file_infos', new_callable=AsyncMock) as soft_delete_file_infos, + patch.object(FileAuditService, 'enqueue_file_audit', new_callable=AsyncMock), + pytest.raises(ServiceException), + ): + asyncio.run( + FileLifecycleService.delete_file_services( + query_db, + make_current_user(), + DeleteFileModel(fileIds=PUBLIC_FILE_ID), + true(), + ) + ) + + assert public_file.read_bytes() == b'public-content' + soft_delete_file_infos.assert_not_awaited() + query_db.rollback.assert_awaited_once() + + +def test_delete_file_restores_physical_file_when_database_commit_fails(tmp_path: Path) -> None: + public_root = tmp_path / 'public' + trash_root = tmp_path / 'trash' + public_file = public_root / 'upload' / 'public.txt' + public_file.parent.mkdir(parents=True) + public_file.write_bytes(b'public-content') + file_infos = [make_file_info(PUBLIC_FILE_ID, 'public', 'upload/public.txt', 'public.txt')] + query_db = make_query_db(commit_error=RuntimeError('commit failed')) + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(public_root)), + patch.object(UploadConfig, 'FILE_TRASH_PATH', str(trash_root)), + patch.object(FileInfoDao, 'get_file_infos_by_ids_for_update', new=AsyncMock(return_value=file_infos)), + patch.object(FileInfoDao, 'soft_delete_file_infos', new_callable=AsyncMock), + pytest.raises(RuntimeError), + ): + asyncio.run( + FileLifecycleService.delete_file_services( + query_db, + make_current_user(), + DeleteFileModel(fileIds=PUBLIC_FILE_ID), + true(), + ) + ) + + assert public_file.read_bytes() == b'public-content' + assert [path for path in trash_root.rglob('*') if path.is_file()] == [] + query_db.rollback.assert_awaited_once() + + +def test_delete_file_rejects_storage_path_escape(tmp_path: Path) -> None: + public_root = tmp_path / 'public' + public_root.mkdir() + outside_file = tmp_path / 'outside.txt' + outside_file.write_bytes(b'outside-content') + file_infos = [make_file_info(PUBLIC_FILE_ID, 'public', '../outside.txt', 'outside.txt')] + query_db = make_query_db() + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(public_root)), + patch.object(FileInfoDao, 'get_file_infos_by_ids_for_update', new=AsyncMock(return_value=file_infos)), + patch.object(FileInfoDao, 'soft_delete_file_infos', new_callable=AsyncMock) as soft_delete_file_infos, + pytest.raises(ServiceException), + ): + asyncio.run( + FileLifecycleService.delete_file_services( + query_db, + make_current_user(), + DeleteFileModel(fileIds=PUBLIC_FILE_ID), + true(), + ) + ) + + assert outside_file.read_bytes() == b'outside-content' + soft_delete_file_infos.assert_not_awaited() + query_db.rollback.assert_awaited_once() + + +def test_delete_file_does_not_overwrite_existing_recycle_bin_file(tmp_path: Path) -> None: + public_root = tmp_path / 'public' + trash_root = tmp_path / 'trash' + public_file = public_root / 'upload' / 'public.txt' + trash_file = trash_root / PUBLIC_FILE_ID / 'public.txt' + public_file.parent.mkdir(parents=True) + trash_file.parent.mkdir(parents=True) + public_file.write_bytes(b'current-content') + trash_file.write_bytes(b'existing-content') + file_infos = [make_file_info(PUBLIC_FILE_ID, 'public', 'upload/public.txt', 'public.txt')] + query_db = make_query_db() + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(public_root)), + patch.object(UploadConfig, 'FILE_TRASH_PATH', str(trash_root)), + patch.object(FileInfoDao, 'get_file_infos_by_ids_for_update', new=AsyncMock(return_value=file_infos)), + patch.object(FileInfoDao, 'soft_delete_file_infos', new_callable=AsyncMock) as soft_delete_file_infos, + pytest.raises(ServiceException), + ): + asyncio.run( + FileLifecycleService.delete_file_services( + query_db, + make_current_user(), + DeleteFileModel(fileIds=PUBLIC_FILE_ID), + true(), + ) + ) + + assert public_file.read_bytes() == b'current-content' + assert trash_file.read_bytes() == b'existing-content' + soft_delete_file_infos.assert_not_awaited() + query_db.rollback.assert_awaited_once() + + +def test_delete_file_restores_already_staged_files_when_later_path_is_invalid(tmp_path: Path) -> None: + public_root = tmp_path / 'public' + valid_file = public_root / 'upload' / 'public.txt' + valid_file.parent.mkdir(parents=True) + valid_file.write_bytes(b'public-content') + outside_file = tmp_path / 'outside.txt' + outside_file.write_bytes(b'outside-content') + file_infos = [ + make_file_info(PUBLIC_FILE_ID, 'public', 'upload/public.txt', 'public.txt'), + make_file_info(PRIVATE_FILE_ID, 'public', '../outside.txt', 'outside.txt'), + ] + query_db = make_query_db() + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(public_root)), + patch.object(FileInfoDao, 'get_file_infos_by_ids_for_update', new=AsyncMock(return_value=file_infos)), + patch.object(FileInfoDao, 'soft_delete_file_infos', new_callable=AsyncMock) as soft_delete_file_infos, + pytest.raises(ServiceException), + ): + asyncio.run( + FileLifecycleService.delete_file_services( + query_db, + make_current_user(), + DeleteFileModel(fileIds=f'{PUBLIC_FILE_ID},{PRIVATE_FILE_ID}'), + true(), + ) + ) + + assert valid_file.read_bytes() == b'public-content' + assert outside_file.read_bytes() == b'outside-content' + soft_delete_file_infos.assert_not_awaited() + query_db.rollback.assert_awaited_once() + + +def test_delete_file_allows_missing_physical_file_to_close_metadata(tmp_path: Path) -> None: + public_root = tmp_path / 'public' + public_root.mkdir() + file_infos = [make_file_info(PUBLIC_FILE_ID, 'public', 'upload/missing.txt', 'missing.txt')] + query_db = make_query_db() + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(public_root)), + patch.object(FileInfoDao, 'get_file_infos_by_ids_for_update', new=AsyncMock(return_value=file_infos)), + patch.object(FileInfoDao, 'soft_delete_file_infos', new_callable=AsyncMock) as soft_delete_file_infos, + ): + result = asyncio.run( + FileLifecycleService.delete_file_services( + query_db, + make_current_user(), + DeleteFileModel(fileIds=PUBLIC_FILE_ID), + true(), + ) + ) + + assert result.is_success is True + soft_delete_file_infos.assert_awaited_once() + query_db.commit.assert_awaited_once() + + +def test_delete_file_rejects_invalid_or_missing_file_ids() -> None: + with pytest.raises(ServiceException): + FileUtil.parse_file_ids('invalid-id') + + query_db = make_query_db() + with ( + patch.object(FileInfoDao, 'get_file_infos_by_ids_for_update', new=AsyncMock(return_value=[])), + pytest.raises(ServiceException), + ): + asyncio.run( + FileLifecycleService.delete_file_services( + query_db, + make_current_user(), + DeleteFileModel(fileIds=PUBLIC_FILE_ID), + true(), + ) + ) + + query_db.rollback.assert_awaited_once() + + +def test_delete_file_rejects_file_outside_data_scope() -> None: + query_db = make_query_db() + file_data_scope_sql = false() + + with ( + patch.object( + FileInfoDao, + 'get_file_infos_by_ids_for_update', + new=AsyncMock(return_value=[]), + ) as get_file_infos, + patch.object(FileInfoDao, 'soft_delete_file_infos', new_callable=AsyncMock) as soft_delete_file_infos, + pytest.raises(ServiceException) as scope_error, + ): + asyncio.run( + FileLifecycleService.delete_file_services( + query_db, + make_current_user(), + DeleteFileModel(fileIds=PUBLIC_FILE_ID), + file_data_scope_sql, + ) + ) + + assert get_file_infos.await_args.args[2] is file_data_scope_sql + assert scope_error.value.message == '部分文件不存在、已删除或超出数据权限' + soft_delete_file_infos.assert_not_awaited() + query_db.rollback.assert_awaited_once() + + +def test_restore_file_moves_both_storage_types_out_of_recycle_bin(tmp_path: Path) -> None: + public_root = tmp_path / 'public' + private_root = tmp_path / 'private' + trash_root = tmp_path / 'trash' + public_trash_file = trash_root / PUBLIC_FILE_ID / 'public.txt' + private_trash_file = trash_root / PRIVATE_FILE_ID / 'private.txt' + public_trash_file.parent.mkdir(parents=True) + private_trash_file.parent.mkdir(parents=True) + public_trash_file.write_bytes(b'public-content') + private_trash_file.write_bytes(b'private-content') + file_infos = [ + make_file_info(PUBLIC_FILE_ID, 'public', 'upload/public.txt', 'public.txt'), + make_file_info(PRIVATE_FILE_ID, 'private', 'upload/private.txt', 'private.txt'), + ] + query_db = make_query_db() + query_db.commit.side_effect = lambda: expire_model_attributes(*file_infos) + move_file = UploadUtil.move_file + + def move_file_after_database_commit(source: Path, target: Path) -> None: + query_db.commit.assert_awaited_once() + move_file(source, target) + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(public_root)), + patch.object(UploadConfig, 'PRIVATE_UPLOAD_PATH', str(private_root)), + patch.object(UploadConfig, 'FILE_TRASH_PATH', str(trash_root)), + patch.object( + FileInfoDao, + 'get_deleted_file_infos_by_ids_for_update', + new=AsyncMock(return_value=file_infos), + ), + patch.object(FileInfoDao, 'restore_file_infos', new_callable=AsyncMock) as restore_file_infos, + patch.object(UploadUtil, 'move_file', side_effect=move_file_after_database_commit), + patch.object(FileAuditService, 'enqueue_file_audit', new_callable=AsyncMock) as enqueue_file_audit, + ): + result = asyncio.run( + FileLifecycleService.restore_file_services( + query_db, + make_current_user(), + f'{PUBLIC_FILE_ID},{PRIVATE_FILE_ID}', + true(), + ) + ) + + assert result.is_success is True + assert (public_root / 'upload' / 'public.txt').read_bytes() == b'public-content' + assert (private_root / 'upload' / 'private.txt').read_bytes() == b'private-content' + assert not public_trash_file.exists() + assert not private_trash_file.exists() + assert restore_file_infos.await_args.args[1] == [PUBLIC_FILE_ID, PRIVATE_FILE_ID] + query_db.commit.assert_awaited_once() + assert enqueue_file_audit.await_count == BATCH_FILE_COUNT + assert [item.args[2:5] for item in enqueue_file_audit.await_args_list] == [ + (PUBLIC_FILE_ID, 'restore', 'completed'), + (PRIVATE_FILE_ID, 'restore', 'completed'), + ] + + +def test_restore_file_returns_physical_file_to_recycle_bin_when_database_commit_fails(tmp_path: Path) -> None: + public_root = tmp_path / 'public' + trash_root = tmp_path / 'trash' + trash_file = trash_root / PUBLIC_FILE_ID / 'public.txt' + trash_file.parent.mkdir(parents=True) + trash_file.write_bytes(b'public-content') + file_infos = [make_file_info(PUBLIC_FILE_ID, 'public', 'upload/public.txt', 'public.txt')] + query_db = make_query_db(commit_error=RuntimeError('commit failed')) + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(public_root)), + patch.object(UploadConfig, 'FILE_TRASH_PATH', str(trash_root)), + patch.object( + FileInfoDao, + 'get_deleted_file_infos_by_ids_for_update', + new=AsyncMock(return_value=file_infos), + ), + patch.object(FileInfoDao, 'restore_file_infos', new_callable=AsyncMock), + pytest.raises(RuntimeError), + ): + asyncio.run( + FileLifecycleService.restore_file_services( + query_db, + make_current_user(), + PUBLIC_FILE_ID, + true(), + ) + ) + + assert trash_file.read_bytes() == b'public-content' + assert not (public_root / 'upload' / 'public.txt').exists() + query_db.rollback.assert_awaited_once() + + +def test_restore_file_compensates_metadata_when_file_move_fails(tmp_path: Path) -> None: + public_root = tmp_path / 'public' + private_root = tmp_path / 'private' + trash_root = tmp_path / 'trash' + public_trash_file = trash_root / PUBLIC_FILE_ID / 'public.txt' + private_trash_file = trash_root / PRIVATE_FILE_ID / 'private.txt' + public_trash_file.parent.mkdir(parents=True) + private_trash_file.parent.mkdir(parents=True) + public_trash_file.write_bytes(b'public-content') + private_trash_file.write_bytes(b'private-content') + file_infos = [ + make_file_info(PUBLIC_FILE_ID, 'public', 'upload/public.txt', 'public.txt'), + make_file_info(PRIVATE_FILE_ID, 'private', 'upload/private.txt', 'private.txt'), + ] + query_db = make_query_db() + move_file = UploadUtil.move_file + move_count = 0 + failed_move_number = 2 + expected_commit_count = 2 + + def fail_second_move(source: Path, target: Path) -> None: + nonlocal move_count + move_count += 1 + if move_count == failed_move_number: + raise OSError('move failed') + move_file(source, target) + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(public_root)), + patch.object(UploadConfig, 'PRIVATE_UPLOAD_PATH', str(private_root)), + patch.object(UploadConfig, 'FILE_TRASH_PATH', str(trash_root)), + patch.object( + FileInfoDao, + 'get_deleted_file_infos_by_ids_for_update', + new=AsyncMock(return_value=file_infos), + ), + patch.object(FileInfoDao, 'restore_file_infos', new_callable=AsyncMock), + patch.object(FileInfoDao, 'soft_delete_file_infos', new_callable=AsyncMock) as soft_delete_file_infos, + patch.object(UploadUtil, 'move_file', side_effect=fail_second_move), + pytest.raises(ServiceException) as restore_error, + ): + asyncio.run( + FileLifecycleService.restore_file_services( + query_db, + make_current_user(), + f'{PUBLIC_FILE_ID},{PRIVATE_FILE_ID}', + true(), + ) + ) + + assert restore_error.value.message == '文件从回收区恢复失败' + assert public_trash_file.read_bytes() == b'public-content' + assert private_trash_file.read_bytes() == b'private-content' + assert not (public_root / 'upload' / 'public.txt').exists() + assert not (private_root / 'upload' / 'private.txt').exists() + soft_delete_file_infos.assert_awaited_once() + assert query_db.commit.await_count == expected_commit_count + + +def test_restore_file_rejects_existing_original_path(tmp_path: Path) -> None: + public_root = tmp_path / 'public' + trash_root = tmp_path / 'trash' + public_file = public_root / 'upload' / 'public.txt' + trash_file = trash_root / PUBLIC_FILE_ID / 'public.txt' + public_file.parent.mkdir(parents=True) + trash_file.parent.mkdir(parents=True) + public_file.write_bytes(b'current-content') + trash_file.write_bytes(b'deleted-content') + file_infos = [make_file_info(PUBLIC_FILE_ID, 'public', 'upload/public.txt', 'public.txt')] + query_db = make_query_db() + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(public_root)), + patch.object(UploadConfig, 'FILE_TRASH_PATH', str(trash_root)), + patch.object( + FileInfoDao, + 'get_deleted_file_infos_by_ids_for_update', + new=AsyncMock(return_value=file_infos), + ), + patch.object(FileInfoDao, 'restore_file_infos', new_callable=AsyncMock) as restore_file_infos, + pytest.raises(ServiceException) as restore_error, + ): + asyncio.run( + FileLifecycleService.restore_file_services( + query_db, + make_current_user(), + PUBLIC_FILE_ID, + true(), + ) + ) + + assert restore_error.value.message == '文件从回收区恢复失败' + assert public_file.read_bytes() == b'current-content' + assert trash_file.read_bytes() == b'deleted-content' + restore_file_infos.assert_not_awaited() + query_db.rollback.assert_awaited_once() + + +def test_restore_file_rejects_file_outside_data_scope() -> None: + query_db = make_query_db() + file_data_scope_sql = false() + + with ( + patch.object( + FileInfoDao, + 'get_deleted_file_infos_by_ids_for_update', + new=AsyncMock(return_value=[]), + ) as get_file_infos, + patch.object(FileInfoDao, 'restore_file_infos', new_callable=AsyncMock) as restore_file_infos, + pytest.raises(ServiceException) as scope_error, + ): + asyncio.run( + FileLifecycleService.restore_file_services( + query_db, + make_current_user(), + PUBLIC_FILE_ID, + file_data_scope_sql, + ) + ) + + assert get_file_infos.await_args.args[2] is file_data_scope_sql + assert scope_error.value.message == '部分文件不存在、未删除或超出数据权限' + restore_file_infos.assert_not_awaited() + query_db.rollback.assert_awaited_once() diff --git a/ruoyi-fastapi-backend/tests/test_file_reconcile.py b/ruoyi-fastapi-backend/tests/test_file_reconcile.py new file mode 100644 index 0000000..2edee72 --- /dev/null +++ b/ruoyi-fastapi-backend/tests/test_file_reconcile.py @@ -0,0 +1,340 @@ +import asyncio +import hashlib +import os +import sys +from datetime import datetime +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +from common.vo import PageModel +from module_admin.dao.file_info_dao import FileInfoDao +from module_admin.entity.vo.file_vo import ( + FileReconcileHandleModel, + FileReconcileIssuePageQueryModel, + FileReconcileRunPageQueryModel, +) +from module_admin.service.file_service import FileReconcileService +from utils.file_util import FileReconcileScanResult, FileReconcileUtil + +FILE_ID = '11111111-1111-4111-8111-111111111111' +STORED_NAME = 'report_20260725120000A001.txt' +STORAGE_KEY = f'2026/07/25/{STORED_NAME}' + + +def make_roots(tmp_path: Path) -> dict[str, Path]: + roots = { + 'public': tmp_path / 'public', + 'private': tmp_path / 'private', + 'trash': tmp_path / 'trash', + 'quarantine': tmp_path / 'quarantine', + } + for root in roots.values(): + root.mkdir() + return roots + + +def make_file_info(content: bytes, **overrides: object) -> dict[str, object]: + file_info: dict[str, object] = { + 'file_id': FILE_ID, + 'storage_type': 'local', + 'access_type': 'public', + 'storage_key': STORAGE_KEY, + 'stored_name': STORED_NAME, + 'file_size': len(content), + 'file_hash': hashlib.sha256(content).hexdigest(), + 'status': 'active', + 'del_flag': '0', + } + file_info.update(overrides) + return file_info + + +def write_file(root: Path, relative_key: str, content: bytes) -> Path: + target = root / Path(relative_key) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + return target + + +def scan( + roots: dict[str, Path], + file_infos: list[dict[str, object]], + check_hash: bool = False, +) -> FileReconcileScanResult: + with patch.object(FileReconcileUtil, 'get_storage_roots', return_value=roots): + return FileReconcileUtil.scan_storage(file_infos, check_hash) + + +def test_reconcile_scan_reports_normal_storage_without_findings(tmp_path: Path) -> None: + roots = make_roots(tmp_path) + content = b'normal file' + write_file(roots['public'], STORAGE_KEY, content) + + result = scan(roots, [make_file_info(content)], check_hash=True) + + assert result.scanned_file_count == 1 + assert result.scanned_storage_count == 1 + assert result.findings == [] + + +def test_reconcile_scan_detects_unexpected_trash_and_orphan_file(tmp_path: Path) -> None: + roots = make_roots(tmp_path) + content = b'trash file' + write_file(roots['trash'], f'{FILE_ID}/{STORED_NAME}', content) + orphan_path = write_file(roots['private'], 'orphan/orphan.txt', b'orphan') + old_time = orphan_path.stat().st_mtime - FileReconcileUtil.ORPHAN_GRACE_SECONDS - 1 + os.utime(orphan_path, (old_time, old_time)) + + result = scan(roots, [make_file_info(content)]) + finding_types = {finding.issue_type for finding in result.findings} + + assert finding_types == {'unexpected_trash', 'orphan_file'} + + +def test_reconcile_scan_detects_wrong_root_without_duplicate_orphan(tmp_path: Path) -> None: + roots = make_roots(tmp_path) + content = b'wrong root' + write_file(roots['private'], STORAGE_KEY, content) + + result = scan(roots, [make_file_info(content)]) + + assert [finding.issue_type for finding in result.findings] == ['wrong_storage_root'] + assert result.findings[0].actual_root == 'private' + assert result.findings[0].expected_root == 'public' + + +def test_reconcile_scan_ignores_recent_orphan_during_upload_window(tmp_path: Path) -> None: + roots = make_roots(tmp_path) + write_file(roots['public'], 'uploading.txt', b'uploading') + + result = scan(roots, []) + + assert result.scanned_storage_count == 1 + assert result.findings == [] + + +def test_reconcile_scan_detects_size_and_hash_mismatch(tmp_path: Path) -> None: + roots = make_roots(tmp_path) + actual_content = b'changed content' + write_file(roots['public'], STORAGE_KEY, actual_content) + file_info = make_file_info(b'old') + + result = scan(roots, [file_info], check_hash=True) + + assert {finding.issue_type for finding in result.findings} == { + 'size_mismatch', + 'hash_mismatch', + } + + +def test_reconcile_scan_rejects_unsafe_metadata_path(tmp_path: Path) -> None: + roots = make_roots(tmp_path) + + result = scan(roots, [make_file_info(b'', storage_key='../outside.txt')]) + + assert len(result.findings) == 1 + assert result.findings[0].issue_type == 'invalid_metadata' + + +def test_reconcile_available_actions_follow_issue_status() -> None: + open_orphan = SimpleNamespace( + status='open', + issue_type='orphan_file', + actual_root='public', + quarantine_key=None, + ) + quarantined = SimpleNamespace( + status='quarantined', + issue_type='orphan_file', + actual_root='public', + quarantine_key='1/public/orphan.txt', + ) + + assert FileReconcileService._get_available_actions(open_orphan) == [ + 'ignore', + 'quarantine_file', + 'register_orphan', + ] + assert FileReconcileService._get_available_actions(quarantined) == [ + 'restore_quarantine', + 'delete_quarantine', + ] + + +def test_reconcile_move_rejects_existing_target(tmp_path: Path) -> None: + roots = make_roots(tmp_path) + write_file(roots['public'], 'source.txt', b'source') + write_file(roots['private'], 'target.txt', b'target') + + with ( + patch.object(FileReconcileUtil, 'get_storage_roots', return_value=roots), + pytest.raises(FileExistsError), + ): + FileReconcileUtil.move_regular_file('public', 'source.txt', 'private', 'target.txt') + + +def test_reconcile_start_creates_manual_run_with_boolean_hash_flag() -> None: + query_db = SimpleNamespace(commit=AsyncMock(), rollback=AsyncMock()) + current_user = SimpleNamespace(user=SimpleNamespace(admin=True, user_name='admin')) + + with ( + patch.object(FileInfoDao, 'release_stale_runs', new=AsyncMock()), + patch.object(FileInfoDao, 'add_reconcile_run', new=AsyncMock()) as add_run, + ): + reconcile_run = asyncio.run( + FileReconcileService.start_reconcile_run_services( + query_db, + check_hash=True, + current_user=current_user, + ) + ) + + assert reconcile_run.check_hash is True + assert reconcile_run.trigger_type == 'manual' + assert reconcile_run.started_by == 'admin' + assert add_run.await_args.args[1].check_hash == '1' + query_db.commit.assert_awaited_once() + + +def test_reconcile_run_page_converts_database_field_names() -> None: + current_user = SimpleNamespace(user=SimpleNamespace(admin=True, user_name='admin')) + run_page = PageModel.model_validate( + { + 'rows': [ + { + 'run_id': FILE_ID, + 'trigger_type': 'manual', + 'status': 'completed', + 'check_hash': '0', + 'started_time': '2026-07-25 12:00:00', + } + ], + 'page_num': 1, + 'page_size': 10, + 'total': 1, + 'has_next': False, + }, + by_name=True, + ) + + with patch.object( + FileInfoDao, + 'get_reconcile_run_list', + new=AsyncMock(return_value=run_page), + ): + result = asyncio.run( + FileReconcileService.get_reconcile_run_list_services( + SimpleNamespace(), + current_user, + FileReconcileRunPageQueryModel(), + ) + ) + + assert isinstance(result, PageModel) + assert result.rows[0].run_id == FILE_ID + assert result.rows[0].check_hash is False + + +def test_reconcile_issue_page_calculates_actions_from_camel_case_rows() -> None: + current_user = SimpleNamespace(user=SimpleNamespace(admin=True, user_name='admin')) + current_time = datetime(2026, 7, 26, 12, 0, 0) + issue_page = PageModel( + rows=[ + { + 'issueId': 1, + 'issueKey': 'orphan:public:orphan.txt', + 'lastRunId': FILE_ID, + 'issueType': 'orphan_file', + 'severity': 'warning', + 'actualRoot': 'public', + 'actualKey': 'orphan.txt', + 'status': 'open', + 'firstSeenTime': current_time, + 'lastSeenTime': current_time, + } + ], + pageNum=1, + pageSize=10, + total=1, + hasNext=False, + ) + + with patch.object( + FileInfoDao, + 'get_reconcile_issue_list', + new=AsyncMock(return_value=issue_page), + ): + result = asyncio.run( + FileReconcileService.get_reconcile_issue_list_services( + SimpleNamespace(), + current_user, + FileReconcileIssuePageQueryModel(), + ) + ) + + assert isinstance(result, PageModel) + assert result.rows[0].available_actions == [ + 'ignore', + 'quarantine_file', + 'register_orphan', + ] + + +def test_reconcile_move_is_compensated_when_database_commit_fails(tmp_path: Path) -> None: + roots = make_roots(tmp_path) + trash_key = f'{FILE_ID}/{STORED_NAME}' + source = write_file(roots['trash'], trash_key, b'content') + target = roots['public'] / Path(STORAGE_KEY) + issue = SimpleNamespace( + issue_id=1, + file_id=FILE_ID, + issue_type='unexpected_trash', + status='open', + actual_root='trash', + actual_key=trash_key, + expected_root='public', + expected_key=STORAGE_KEY, + quarantine_key=None, + handle_action=None, + handle_reason=None, + handled_by=None, + handled_time=None, + ) + query_db = SimpleNamespace( + commit=AsyncMock(side_effect=RuntimeError('commit failed')), + rollback=AsyncMock(), + ) + current_user = SimpleNamespace(user=SimpleNamespace(admin=True, user_name='admin', user_id=1, dept_id=100)) + handle = FileReconcileHandleModel(action='restore_source', reason='恢复事务中断文件') + + with ( + patch.object(FileReconcileUtil, 'get_storage_roots', return_value=roots), + patch.object( + FileInfoDao, + 'has_running_reconcile_run', + new=AsyncMock(return_value=False), + ), + patch.object( + FileInfoDao, + 'get_reconcile_issue_for_update', + new=AsyncMock(return_value=issue), + ), + pytest.raises(RuntimeError, match='commit failed'), + ): + asyncio.run( + FileReconcileService.handle_reconcile_issue_services( + query_db, + current_user, + issue.issue_id, + handle, + ) + ) + + assert source.is_file() + assert not target.exists() + query_db.rollback.assert_awaited_once() diff --git a/ruoyi-fastapi-backend/tests/test_file_reference.py b/ruoyi-fastapi-backend/tests/test_file_reference.py new file mode 100644 index 0000000..31cd71f --- /dev/null +++ b/ruoyi-fastapi-backend/tests/test_file_reference.py @@ -0,0 +1,291 @@ +import asyncio +import os +import sys +from datetime import datetime, timedelta +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest +from sqlalchemy import true + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +from exceptions.exception import ServiceException +from module_admin.dao.file_business_dao import FileReferenceDao +from module_admin.dao.file_info_dao import FileInfoDao +from module_admin.entity.do.file_do import SysFileReference +from module_admin.entity.vo.file_vo import FileRetentionPolicyModel +from module_admin.service.file_business_service import FileReferenceService, FileRetentionPolicyService + +FILE_ID = '11111111-1111-4111-8111-111111111111' + + +def make_query_db() -> SimpleNamespace: + return SimpleNamespace(commit=AsyncMock(), rollback=AsyncMock()) + + +def test_get_file_reference_list_includes_legacy_reference() -> None: + file_info = { + 'file_id': FILE_ID, + 'business_type': 'notice', + 'business_id': '10', + } + with ( + patch.object( + FileInfoDao, + 'get_file_management_detail_by_id', + new=AsyncMock(return_value=file_info), + ), + patch.object(FileReferenceDao, 'get_file_reference_list', new=AsyncMock(return_value=[])), + ): + result = asyncio.run( + FileReferenceService.get_file_reference_list_services( + make_query_db(), + FILE_ID, + true(), + ) + ) + + assert len(result) == 1 + assert result[0].business_type == 'notice' + assert result[0].business_id == '10' + assert result[0].legacy is True + + +def test_get_file_reference_list_converts_orm_reference() -> None: + create_time = datetime(2026, 7, 26, 10, 0, 0) + file_info = { + 'file_id': FILE_ID, + 'business_type': None, + 'business_id': None, + } + file_reference = SysFileReference( + reference_id=1, + file_id=FILE_ID, + business_type='contract', + business_id='100', + business_name='Contract', + create_by='admin', + create_time=create_time, + ) + with ( + patch.object( + FileInfoDao, + 'get_file_management_detail_by_id', + new=AsyncMock(return_value=file_info), + ), + patch.object( + FileReferenceDao, + 'get_file_reference_list', + new=AsyncMock(return_value=[file_reference]), + ), + ): + result = asyncio.run( + FileReferenceService.get_file_reference_list_services( + make_query_db(), + FILE_ID, + true(), + ) + ) + + assert result[0].model_dump(by_alias=True) == { + 'referenceId': 1, + 'fileId': FILE_ID, + 'businessType': 'contract', + 'businessId': '100', + 'businessName': 'Contract', + 'retentionExpireTime': None, + 'createBy': 'admin', + 'createTime': create_time, + 'legacy': False, + } + + +def test_replace_business_file_references_locks_files_without_committing() -> None: + query_db = make_query_db() + file_infos = [SimpleNamespace(file_id=FILE_ID)] + with ( + patch.object( + FileInfoDao, + 'get_file_infos_by_ids_for_update', + new=AsyncMock(return_value=file_infos), + ) as get_file_infos, + patch.object( + FileReferenceDao, + 'replace_business_file_references', + new_callable=AsyncMock, + ) as replace_references, + patch.object( + FileRetentionPolicyService, + 'get_enabled_file_retention_policy_services', + new=AsyncMock(return_value=None), + ), + ): + asyncio.run( + FileReferenceService.replace_business_file_references_services( + query_db, + 'notice', + '10', + [FILE_ID, FILE_ID], + create_by='admin', + file_data_scope_sql=true(), + business_name='系统公告', + ) + ) + + assert get_file_infos.await_args.args[1] == [FILE_ID] + reference_list = replace_references.await_args.args[3] + assert len(reference_list) == 1 + assert isinstance(reference_list[0], SysFileReference) + assert reference_list[0].file_id == FILE_ID + assert reference_list[0].business_type == 'notice' + assert reference_list[0].business_id == '10' + assert reference_list[0].business_name == '系统公告' + query_db.commit.assert_not_awaited() + + +def test_replace_business_file_references_rejects_invalid_file() -> None: + query_db = make_query_db() + with ( + patch.object(FileInfoDao, 'get_file_infos_by_ids_for_update', new=AsyncMock(return_value=[])), + patch.object( + FileReferenceDao, + 'replace_business_file_references', + new_callable=AsyncMock, + ) as replace_references, + pytest.raises(ServiceException) as file_error, + ): + asyncio.run( + FileReferenceService.replace_business_file_references_services( + query_db, + 'notice', + '10', + [FILE_ID], + create_by='admin', + file_data_scope_sql=true(), + ) + ) + + assert file_error.value.message == '部分引用文件不存在或已失效' + replace_references.assert_not_awaited() + + +def test_remove_business_file_references_does_not_lock_files() -> None: + query_db = make_query_db() + with ( + patch.object( + FileReferenceDao, + 'replace_business_file_references', + new_callable=AsyncMock, + ) as replace_references, + patch.object( + FileRetentionPolicyService, + 'get_enabled_file_retention_policy_services', + new=AsyncMock(return_value=None), + ), + ): + asyncio.run( + FileReferenceService.remove_business_file_references_services( + query_db, + 'notice', + '10', + ) + ) + + assert replace_references.await_args.args[3] == [] + query_db.commit.assert_not_awaited() + + +def test_replace_business_file_references_applies_retention_policy() -> None: + query_db = make_query_db() + create_time = datetime(2026, 7, 23, 10, 0, 0) + policy = FileRetentionPolicyModel(businessType='notice', retentionDays=30) + with ( + patch.object( + FileInfoDao, + 'get_file_infos_by_ids_for_update', + new=AsyncMock(return_value=[SimpleNamespace(file_id=FILE_ID)]), + ), + patch.object( + FileRetentionPolicyService, + 'get_enabled_file_retention_policy_services', + new=AsyncMock(return_value=policy), + ), + patch.object( + FileReferenceDao, + 'replace_business_file_references', + new_callable=AsyncMock, + ) as replace_references, + patch( + 'module_admin.service.file_business_service.datetime', + new=SimpleNamespace(now=lambda: create_time), + ), + ): + asyncio.run( + FileReferenceService.replace_business_file_references_services( + query_db, + 'notice', + '10', + [FILE_ID], + create_by='admin', + file_data_scope_sql=true(), + ) + ) + + reference = replace_references.await_args.args[3][0] + assert reference.retention_expire_time == create_time + timedelta(days=30) + + +def test_replace_business_file_references_preserves_extended_expiration() -> None: + policy_expire_time = datetime(2026, 8, 22, 10, 0, 0) + extended_expire_time = datetime(2027, 7, 23, 10, 0, 0) + old_reference = SysFileReference( + file_id=FILE_ID, + business_type='notice', + business_id='10', + retention_expire_time=extended_expire_time, + ) + new_reference = SysFileReference( + file_id=FILE_ID, + business_type='notice', + business_id='10', + retention_expire_time=policy_expire_time, + ) + + FileReferenceDao._preserve_later_retention_expire_times( + [old_reference], + [new_reference], + ) + + assert new_reference.retention_expire_time == extended_expire_time + + +def test_refresh_file_expire_times_uses_latest_reference_expiration() -> None: + first_expire_time = datetime(2026, 8, 1) + last_expire_time = datetime(2026, 9, 1) + file_info = SimpleNamespace( + file_id=FILE_ID, + business_type=None, + business_id=None, + expire_time=None, + ) + query_db = SimpleNamespace( + execute=AsyncMock( + return_value=SimpleNamespace( + all=lambda: [ + (FILE_ID, first_expire_time), + (FILE_ID, last_expire_time), + ] + ) + ) + ) + + asyncio.run( + FileReferenceDao._refresh_file_expire_times( + query_db, + [FILE_ID], + {FILE_ID: file_info}, + ) + ) + + assert file_info.expire_time == last_expire_time diff --git a/ruoyi-fastapi-backend/tests/test_file_retention_policy.py b/ruoyi-fastapi-backend/tests/test_file_retention_policy.py new file mode 100644 index 0000000..22f12d4 --- /dev/null +++ b/ruoyi-fastapi-backend/tests/test_file_retention_policy.py @@ -0,0 +1,130 @@ +import asyncio +import os +import sys +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +from exceptions.exception import ServiceException +from module_admin.dao.file_business_dao import FileRetentionPolicyDao +from module_admin.entity.do.file_do import SysFileRetentionPolicy +from module_admin.entity.vo.file_vo import FileRetentionPolicyModel +from module_admin.service.file_business_service import FileRetentionPolicyService + +RETENTION_DAYS = 365 + + +def make_query_db() -> SimpleNamespace: + return SimpleNamespace(commit=AsyncMock(), rollback=AsyncMock()) + + +def test_get_file_retention_policy_list_maps_orm_field_names() -> None: + query_db = make_query_db() + db_policy = SysFileRetentionPolicy( + business_type='notice', + retention_days=RETENTION_DAYS, + status='0', + remark='公告附件', + ) + with patch.object( + FileRetentionPolicyDao, + 'get_file_retention_policy_list', + new=AsyncMock(return_value=[db_policy]), + ): + result = asyncio.run(FileRetentionPolicyService.get_file_retention_policy_list_services(query_db)) + + assert len(result) == 1 + assert result[0].business_type == 'notice' + assert result[0].retention_days == RETENTION_DAYS + + +def test_get_enabled_file_retention_policy_maps_orm_field_names() -> None: + query_db = make_query_db() + db_policy = SysFileRetentionPolicy( + business_type='notice', + retention_days=RETENTION_DAYS, + status='0', + ) + with patch.object( + FileRetentionPolicyDao, + 'get_file_retention_policy_by_business_type', + new=AsyncMock(return_value=db_policy), + ) as get_policy: + result = asyncio.run( + FileRetentionPolicyService.get_enabled_file_retention_policy_services( + query_db, + 'notice', + ) + ) + + assert result is not None + assert result.business_type == 'notice' + assert result.retention_days == RETENTION_DAYS + get_policy.assert_awaited_once_with(query_db, 'notice', enabled_only=True) + + +def test_add_file_retention_policy_commits_new_policy() -> None: + query_db = make_query_db() + policy = FileRetentionPolicyModel( + businessType='notice', + retentionDays=RETENTION_DAYS, + status='0', + remark='公告附件', + ) + with ( + patch.object( + FileRetentionPolicyDao, + 'get_file_retention_policy_by_business_type', + new=AsyncMock(return_value=None), + ), + patch.object( + FileRetentionPolicyDao, + 'add_file_retention_policy', + new_callable=AsyncMock, + ) as add_policy, + ): + result = asyncio.run( + FileRetentionPolicyService.add_file_retention_policy_services( + query_db, + policy, + 'admin', + ) + ) + + assert result.is_success is True + assert add_policy.await_args.args[1].business_type == 'notice' + assert add_policy.await_args.args[1].retention_days == RETENTION_DAYS + assert add_policy.await_args.args[1].create_by == 'admin' + query_db.commit.assert_awaited_once() + + +def test_add_file_retention_policy_rejects_duplicate_business_type() -> None: + query_db = make_query_db() + policy = FileRetentionPolicyModel(businessType='notice', retentionDays=30) + with ( + patch.object( + FileRetentionPolicyDao, + 'get_file_retention_policy_by_business_type', + new=AsyncMock(return_value=SimpleNamespace(business_type='notice')), + ), + patch.object( + FileRetentionPolicyDao, + 'add_file_retention_policy', + new_callable=AsyncMock, + ) as add_policy, + pytest.raises(ServiceException) as policy_error, + ): + asyncio.run( + FileRetentionPolicyService.add_file_retention_policy_services( + query_db, + policy, + 'admin', + ) + ) + + assert policy_error.value.message == '业务类型notice的保留策略已存在' + add_policy.assert_not_awaited() + query_db.commit.assert_not_awaited() diff --git a/ruoyi-fastapi-backend/tests/test_migrate_legacy_files.py b/ruoyi-fastapi-backend/tests/test_migrate_legacy_files.py new file mode 100644 index 0000000..72afba8 --- /dev/null +++ b/ruoyi-fastapi-backend/tests/test_migrate_legacy_files.py @@ -0,0 +1,169 @@ +import asyncio +import os +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +from config.env import UploadConfig +from module_admin.dao.file_info_dao import FileInfoDao +from module_admin.entity.do.file_do import SysFileInfo +from module_admin.entity.vo.file_vo import FileInfoModel +from scripts.migrate_legacy_files import calculate_file_hash, collect_legacy_files, migrate_legacy_files + +EXPECTED_VALIDATION_LOOKUP_COUNT = 2 +EXPECTED_BATCH_FILE_COUNT = 3 +EXPECTED_BATCH_COMMIT_COUNT = 2 + + +class AsyncSessionContext: + """ + 测试用异步数据库会话上下文 + """ + + def __init__(self, session: SimpleNamespace) -> None: + self.session = session + + async def __aenter__(self) -> SimpleNamespace: + return self.session + + async def __aexit__(self, exc_type: type | None, exc_value: BaseException | None, traceback: object) -> None: + return None + + +def test_collect_legacy_files_filters_disallowed_extensions(tmp_path: Path) -> None: + allowed_file = tmp_path / 'upload' / 'report.txt' + allowed_file.parent.mkdir() + allowed_file.write_bytes(b'report-content') + (tmp_path / 'danger.exe').write_bytes(b'danger-content') + + with patch.object(UploadConfig, 'UPLOAD_PATH', str(tmp_path)): + legacy_files, skipped_count = collect_legacy_files() + + assert skipped_count == 1 + assert len(legacy_files) == 1 + assert legacy_files[0].storage_key == 'upload/report.txt' + assert legacy_files[0].file_size == len(b'report-content') + assert asyncio.run(calculate_file_hash(allowed_file, legacy_files[0].signature)) == ( + '362636f5a34836946783f440c823fd9b5604a42a3deebe7b9bb97dfc1084f6ed' + ) + + +def test_collect_legacy_files_filters_oversized_files(tmp_path: Path) -> None: + (tmp_path / 'small.txt').write_bytes(b'ok') + (tmp_path / 'large.txt').write_bytes(b'too-large') + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(tmp_path)), + patch.object(UploadConfig, 'MAX_FILE_SIZE', 2), + ): + legacy_files, skipped_count = collect_legacy_files() + + assert [legacy_file.filename for legacy_file in legacy_files] == ['small.txt'] + assert skipped_count == 1 + + +def test_calculate_file_hash_rejects_changed_file(tmp_path: Path) -> None: + filepath = tmp_path / 'report.txt' + filepath.write_bytes(b'old-content') + with patch.object(UploadConfig, 'UPLOAD_PATH', str(tmp_path)): + legacy_files, skipped_count = collect_legacy_files() + filepath.write_bytes(b'new-content-with-different-size') + + with pytest.raises(ValueError, match='文件在扫描后发生变化'): + asyncio.run(calculate_file_hash(filepath, legacy_files[0].signature)) + + assert skipped_count == 0 + + +def test_migrate_legacy_files_dry_run_performs_full_validation_without_commit(tmp_path: Path) -> None: + (tmp_path / 'report.txt').write_bytes(b'report-content') + session = SimpleNamespace(commit=AsyncMock()) + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(tmp_path)), + patch('scripts.migrate_legacy_files.AsyncSessionLocal', return_value=AsyncSessionContext(session)), + patch.object(FileInfoDao, 'get_file_info_by_storage_key', new=AsyncMock(return_value=None)) as get_file_info, + patch.object(FileInfoDao, 'add_file_info_dao', new_callable=AsyncMock) as add_file_info, + ): + result = asyncio.run(migrate_legacy_files(dry_run=True)) + + assert result == (1, 0) + assert get_file_info.await_count == EXPECTED_VALIDATION_LOOKUP_COUNT + add_file_info.assert_not_awaited() + session.commit.assert_not_awaited() + + +def test_migrate_legacy_files_writes_file_info_model_and_commits(tmp_path: Path) -> None: + (tmp_path / 'report.txt').write_bytes(b'report-content') + session = SimpleNamespace(commit=AsyncMock()) + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(tmp_path)), + patch('scripts.migrate_legacy_files.AsyncSessionLocal', return_value=AsyncSessionContext(session)), + patch.object(FileInfoDao, 'get_file_info_by_storage_key', new=AsyncMock(return_value=None)), + patch.object(FileInfoDao, 'add_file_info_dao', new_callable=AsyncMock) as add_file_info, + ): + result = asyncio.run(migrate_legacy_files(batch_size=1, maintenance_confirmed=True)) + + assert result == (1, 0) + file_info = add_file_info.await_args.args[1] + assert isinstance(file_info, FileInfoModel) + assert file_info.storage_type == 'local' + assert file_info.access_type == 'public' + assert file_info.upload_user_id is None + assert file_info.owner_user_id is None + assert file_info.dept_id is None + assert file_info.file_hash == '362636f5a34836946783f440c823fd9b5604a42a3deebe7b9bb97dfc1084f6ed' + assert SysFileInfo(**file_info.model_dump()).storage_key == 'report.txt' + session.commit.assert_awaited_once() + + +def test_migrate_legacy_files_skips_existing_storage_location(tmp_path: Path) -> None: + (tmp_path / 'report.txt').write_bytes(b'report-content') + session = SimpleNamespace(commit=AsyncMock()) + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(tmp_path)), + patch('scripts.migrate_legacy_files.AsyncSessionLocal', return_value=AsyncSessionContext(session)), + patch.object( + FileInfoDao, + 'get_file_info_by_storage_key', + new=AsyncMock(return_value=SimpleNamespace(file_id='existing')), + ), + patch.object(FileInfoDao, 'add_file_info_dao', new_callable=AsyncMock) as add_file_info, + ): + result = asyncio.run(migrate_legacy_files(maintenance_confirmed=True)) + + assert result == (0, 1) + add_file_info.assert_not_awaited() + session.commit.assert_not_awaited() + + +def test_migrate_legacy_files_commits_by_batch(tmp_path: Path) -> None: + for index in range(3): + (tmp_path / f'report-{index}.txt').write_bytes(f'content-{index}'.encode()) + session = SimpleNamespace(commit=AsyncMock()) + + with ( + patch.object(UploadConfig, 'UPLOAD_PATH', str(tmp_path)), + patch('scripts.migrate_legacy_files.AsyncSessionLocal', return_value=AsyncSessionContext(session)), + patch.object(FileInfoDao, 'get_file_info_by_storage_key', new=AsyncMock(return_value=None)), + patch.object(FileInfoDao, 'add_file_info_dao', new_callable=AsyncMock) as add_file_info, + ): + result = asyncio.run(migrate_legacy_files(batch_size=2, maintenance_confirmed=True)) + + assert result == (3, 0) + assert add_file_info.await_count == EXPECTED_BATCH_FILE_COUNT + assert session.commit.await_count == EXPECTED_BATCH_COMMIT_COUNT + + +def test_migrate_legacy_files_requires_valid_execution_options() -> None: + with pytest.raises(ValueError, match='每批提交数量必须大于0'): + asyncio.run(migrate_legacy_files(dry_run=True, batch_size=0)) + with pytest.raises(ValueError, match='必须停止公开文件上传'): + asyncio.run(migrate_legacy_files()) diff --git a/ruoyi-fastapi-backend/utils/file_util.py b/ruoyi-fastapi-backend/utils/file_util.py new file mode 100644 index 0000000..df97d30 --- /dev/null +++ b/ruoyi-fastapi-backend/utils/file_util.py @@ -0,0 +1,1019 @@ +import hashlib +import os +import re +import shutil +import time +import uuid +from collections.abc import AsyncGenerator, Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol + +from config.env import UploadConfig +from exceptions.exception import FileRangeNotSatisfiableException, ServiceException +from utils.log_util import logger +from utils.upload_util import FilePathUtil, UploadUtil + + +class FileStorageInfo(Protocol): + """ + 文件存储信息协议 + """ + + file_id: str + storage_type: str + access_type: str + storage_key: str + stored_name: str + + +@dataclass(frozen=True) +class FileByteRange: + """ + 文件字节范围 + """ + + start: int + end: int + file_size: int + is_partial: bool + + @property + def length(self) -> int: + """ + 获取范围字节数 + + :return: 范围字节数 + """ + return max(self.end - self.start + 1, 0) + + +@dataclass(frozen=True) +class FileDownloadResult: + """ + 文件下载结果 + """ + + data: AsyncGenerator[bytes, None] + filename: str + byte_range: FileByteRange + accept_ranges: bool = True + + +@dataclass(frozen=True) +class StagedFile: + """ + 待删除文件暂存信息 + """ + + source_path: Path + trash_path: Path + + +@dataclass(frozen=True) +class FileReconcileFinding: + """ + 文件存储对账异常 + """ + + issue_key: str + issue_type: str + severity: str + detail: str + file_id: str | None = None + storage_type: str | None = None + access_type: str | None = None + expected_root: str | None = None + expected_key: str | None = None + actual_root: str | None = None + actual_key: str | None = None + expected_size: int | None = None + actual_size: int | None = None + expected_hash: str | None = None + actual_hash: str | None = None + + +@dataclass(frozen=True) +class FileReconcileScanResult: + """ + 文件存储对账扫描结果 + """ + + findings: list[FileReconcileFinding] + scanned_file_count: int + scanned_storage_count: int + + +@dataclass(frozen=True) +class _FileReconcileContext: + """ + 文件存储对账内部上下文 + """ + + file_info: dict[str, Any] + source_root: str + source_key: str + source_path: Path + trash_key: str + trash_path: Path + + +class FileUtil: + """ + 文件工具类 + """ + + FILE_RANGE_PATTERN = re.compile(r'^bytes=(\d*)-(\d*)$', re.IGNORECASE) + + @classmethod + def parse_byte_range(cls, range_header: str | None, file_size: int) -> FileByteRange: + """ + 解析HTTP单区间Range请求头 + + :param range_header: Range请求头 + :param file_size: 文件总大小 + :return: 文件字节范围 + """ + if file_size < 0: + raise ValueError('文件大小不能小于0') + if not range_header: + return FileByteRange( + start=0, + end=file_size - 1, + file_size=file_size, + is_partial=False, + ) + + range_match = cls.FILE_RANGE_PATTERN.fullmatch(range_header.strip()) + if range_match is None or file_size == 0: + raise FileRangeNotSatisfiableException(file_size) + + start_text, end_text = range_match.groups() + if not start_text and not end_text: + raise FileRangeNotSatisfiableException(file_size) + + if not start_text: + suffix_length = int(end_text) + if suffix_length <= 0: + raise FileRangeNotSatisfiableException(file_size) + start = max(file_size - suffix_length, 0) + end = file_size - 1 + else: + start = int(start_text) + if start >= file_size: + raise FileRangeNotSatisfiableException(file_size) + end = file_size - 1 if not end_text else min(int(end_text), file_size - 1) + if end < start: + raise FileRangeNotSatisfiableException(file_size) + + return FileByteRange( + start=start, + end=end, + file_size=file_size, + is_partial=True, + ) + + @classmethod + def parse_file_ids(cls, file_ids: str) -> list[str]: + """ + 解析并校验文件ID + + :param file_ids: 文件ID字符串 + :return: 文件ID列表 + """ + if not file_ids: + raise ServiceException(message='文件ID不能为空') + try: + parsed_file_ids = list( + dict.fromkeys(str(uuid.UUID(item.strip())) for item in file_ids.split(',') if item.strip()) + ) + except ValueError as exc: + raise ServiceException(message='文件ID格式不正确') from exc + if not parsed_file_ids: + raise ServiceException(message='文件ID不能为空') + return parsed_file_ids + + @classmethod + def enrich_storage_status(cls, file_rows: list[dict[str, Any]]) -> None: + """ + 补充文件物理存储状态 + + :param file_rows: 文件信息列表 + :return: None + """ + for file_row in file_rows: + file_row['storageStatus'] = cls.get_storage_status(file_row) + + @classmethod + def get_storage_status(cls, file_info: dict[str, Any]) -> str: + """ + 获取文件物理存储状态 + + :param file_info: 文件信息 + :return: 文件物理存储状态 + """ + + def get_value(snake_name: str, camel_name: str) -> Any: + return file_info.get(snake_name) if snake_name in file_info else file_info.get(camel_name) + + try: + file_id = get_value('file_id', 'fileId') + storage_type = get_value('storage_type', 'storageType') + access_type = get_value('access_type', 'accessType') + storage_key = get_value('storage_key', 'storageKey') + stored_name = get_value('stored_name', 'storedName') + status = get_value('status', 'status') + if storage_type != 'local' or access_type not in {'public', 'private'}: + return 'invalid' + storage_root = UploadConfig.UPLOAD_PATH if access_type == 'public' else UploadConfig.PRIVATE_UPLOAD_PATH + source_path = FilePathUtil.resolve_path_within_root(storage_root, storage_key) + trash_path = FilePathUtil.resolve_path_within_root( + UploadConfig.FILE_TRASH_PATH, + f'{file_id}/{stored_name}', + ) + source_exists = source_path.exists() + trash_exists = trash_path.exists() + if source_exists and not source_path.is_file(): + return 'invalid' + if trash_exists and not trash_path.is_file(): + return 'invalid' + if status in {'deleted', 'purging'}: + if trash_exists and not source_exists: + return 'quarantined' + return 'invalid' if source_exists else 'missing' + if source_exists and not trash_exists: + return 'normal' + if trash_exists and not source_exists: + return 'quarantined' + return 'invalid' if source_exists and trash_exists else 'missing' + except (AttributeError, OSError, TypeError, ValueError): + return 'invalid' + + @classmethod + def stage_file_deletions(cls, file_infos: list[FileStorageInfo]) -> list[StagedFile]: + """ + 将待删除文件移入回收区 + + :param file_infos: 文件信息列表 + :return: 暂存文件列表 + """ + staged_files = [] + try: + for file_info in file_infos: + staged_file = cls._get_staged_file_paths(file_info) + if not staged_file.source_path.exists(): + if staged_file.trash_path.exists(): + if not staged_file.trash_path.is_file(): + raise ValueError('回收区路径不是普通文件') + staged_files.append(staged_file) + continue + if not staged_file.source_path.is_file(): + raise ValueError('待删除路径不是普通文件') + if staged_file.trash_path.exists(): + raise FileExistsError('回收区目标文件已存在') + UploadUtil.move_file(staged_file.source_path, staged_file.trash_path) + staged_files.append(staged_file) + except Exception: + cls.restore_staged_files(staged_files) + raise + return staged_files + + @classmethod + def _get_staged_file_paths(cls, file_info: FileStorageInfo) -> StagedFile: + """ + 获取文件原路径和回收区路径 + + :param file_info: 文件信息 + :return: 文件暂存信息 + """ + if file_info.storage_type != 'local' or file_info.access_type not in {'public', 'private'}: + raise ValueError('文件存储类型或访问类型异常') + storage_root = ( + UploadConfig.UPLOAD_PATH if file_info.access_type == 'public' else UploadConfig.PRIVATE_UPLOAD_PATH + ) + source_path = FilePathUtil.resolve_path_within_root(storage_root, file_info.storage_key) + trash_path = FilePathUtil.resolve_path_within_root( + UploadConfig.FILE_TRASH_PATH, + f'{file_info.file_id}/{file_info.stored_name}', + ) + return StagedFile(source_path=source_path, trash_path=trash_path) + + @classmethod + def restore_staged_files(cls, staged_files: list[StagedFile]) -> None: + """ + 将隔离区文件恢复到原路径 + + :param staged_files: 暂存文件列表 + :return: None + """ + for staged_file in reversed(staged_files): + try: + if staged_file.trash_path.exists(): + if staged_file.source_path.exists(): + logger.error(f'文件删除回滚失败，原路径已存在: {staged_file.source_path}') + continue + UploadUtil.move_file(staged_file.trash_path, staged_file.source_path) + UploadUtil.remove_empty_directory(staged_file.trash_path.parent) + except OSError as exc: + logger.error(f'文件删除回滚失败: {exc}') + + @classmethod + def prepare_deleted_files_for_restore(cls, file_infos: list[FileStorageInfo]) -> list[StagedFile]: + """ + 校验待恢复文件并生成暂存信息 + + :param file_infos: 文件信息列表 + :return: 暂存文件列表 + """ + staged_files = [] + for file_info in file_infos: + staged_file = cls._get_staged_file_paths(file_info) + if not staged_file.trash_path.exists() or not staged_file.trash_path.is_file(): + raise FileNotFoundError('回收区文件不存在') + if staged_file.source_path.exists(): + raise FileExistsError('文件原路径已存在') + staged_files.append(staged_file) + return staged_files + + @classmethod + def prepare_deleted_files_for_purge(cls, file_infos: list[FileStorageInfo]) -> list[StagedFile]: + """ + 校验待永久清理文件并生成暂存信息 + + :param file_infos: 文件信息列表 + :return: 暂存文件列表 + """ + staged_files = [] + for file_info in file_infos: + staged_file = cls._get_staged_file_paths(file_info) + if staged_file.source_path.exists(): + raise FileExistsError('待清理文件仍存在于正式存储目录') + if staged_file.trash_path.exists() and not staged_file.trash_path.is_file(): + raise ValueError('回收区路径不是普通文件') + staged_files.append(staged_file) + return staged_files + + @classmethod + def purge_deleted_files(cls, staged_files: list[StagedFile]) -> None: + """ + 永久删除回收区文件 + + :param staged_files: 暂存文件列表 + :return: None + """ + for staged_file in staged_files: + if staged_file.source_path.exists(): + raise FileExistsError('待清理文件仍存在于正式存储目录') + if staged_file.trash_path.exists(): + if not staged_file.trash_path.is_file(): + raise ValueError('回收区路径不是普通文件') + staged_file.trash_path.unlink() + UploadUtil.remove_empty_directory(staged_file.trash_path.parent) + + @classmethod + def restore_deleted_files(cls, staged_files: list[StagedFile]) -> None: + """ + 将已删除文件从回收区恢复到原路径 + + :param staged_files: 暂存文件列表 + :return: None + """ + restored_files = [] + try: + for staged_file in staged_files: + if not staged_file.trash_path.exists() or not staged_file.trash_path.is_file(): + raise FileNotFoundError('回收区文件不存在') + if staged_file.source_path.exists(): + raise FileExistsError('文件原路径已存在') + UploadUtil.move_file(staged_file.trash_path, staged_file.source_path) + restored_files.append(staged_file) + except Exception: + cls._restage_restored_files(restored_files) + raise + + @classmethod + def _restage_restored_files(cls, staged_files: list[StagedFile]) -> None: + """ + 将恢复失败的文件重新移入回收区 + + :param staged_files: 暂存文件列表 + :return: None + """ + for staged_file in reversed(staged_files): + try: + if staged_file.source_path.exists(): + if staged_file.trash_path.exists(): + logger.error(f'文件恢复回滚失败，回收区路径已存在: {staged_file.trash_path}') + continue + UploadUtil.move_file(staged_file.source_path, staged_file.trash_path) + except OSError as exc: + logger.error(f'文件恢复回滚失败: {exc}') + + @classmethod + def cleanup_trash_directories(cls, staged_files: list[StagedFile]) -> None: + """ + 清理恢复后留下的空回收区目录 + + :param staged_files: 暂存文件列表 + :return: None + """ + for staged_file in staged_files: + UploadUtil.remove_empty_directory(staged_file.trash_path.parent) + + +class FileReconcileUtil: + """ + 文件存储对账工具类 + """ + + MAX_SCAN_ENTRIES = 1_000_000 + HASH_CHUNK_SIZE = 1024 * 1024 + ORPHAN_GRACE_SECONDS = 300 + + @classmethod + def scan_storage(cls, file_infos: list[dict[str, Any]], check_hash: bool = False) -> FileReconcileScanResult: + """ + 执行数据库和本地文件系统双向对账 + + :param file_infos: 文件信息列表 + :param check_hash: 是否校验文件SHA-256 + :return: 对账扫描结果 + """ + roots = cls.get_storage_roots() + cls._validate_storage_roots(roots) + findings: list[FileReconcileFinding] = [] + contexts: list[_FileReconcileContext] = [] + expected_locations: set[tuple[str, str]] = set() + + for file_info in file_infos: + try: + context = cls._build_file_context(file_info, roots) + except (AttributeError, OSError, TypeError, ValueError) as exc: + findings.append( + cls.build_finding( + issue_type='invalid_metadata', + severity='critical', + detail=f'文件存储元数据不合法：{exc}', + file_id=cls._get_text_value(file_info, 'file_id'), + storage_type=cls._get_text_value(file_info, 'storage_type'), + access_type=cls._get_text_value(file_info, 'access_type'), + expected_key=cls._get_text_value(file_info, 'storage_key'), + ) + ) + continue + contexts.append(context) + expected_locations.add((context.source_root, context.source_key)) + expected_locations.add(('trash', context.trash_key)) + + claimed_unexpected_locations: set[tuple[str, str]] = set() + for context in contexts: + context_findings, claimed_locations = cls._inspect_file_context( + context, + roots, + expected_locations, + check_hash, + ) + findings.extend(context_findings) + claimed_unexpected_locations.update(claimed_locations) + + scanned_storage_count = 0 + for root_name in ('public', 'private', 'trash'): + root_path = roots[root_name] + for relative_key, physical_path, is_unsafe in cls._iter_storage_entries(root_path): + scanned_storage_count += 1 + if scanned_storage_count > cls.MAX_SCAN_ENTRIES: + raise RuntimeError(f'物理文件数量超过扫描上限{cls.MAX_SCAN_ENTRIES}') + location = (root_name, relative_key) + if is_unsafe: + findings.append( + cls.build_finding( + issue_type='unsafe_entry', + severity='critical', + detail='存储目录中存在符号链接或非普通文件条目', + actual_root=root_name, + actual_key=relative_key, + ) + ) + elif location not in expected_locations and location not in claimed_unexpected_locations: + try: + physical_stat = physical_path.stat() + except FileNotFoundError: + continue + if time.time() - physical_stat.st_mtime < cls.ORPHAN_GRACE_SECONDS: + continue + findings.append( + cls.build_finding( + issue_type='orphan_file', + severity='warning', + detail='物理文件未登记到文件信息表', + actual_root=root_name, + actual_key=relative_key, + actual_size=physical_stat.st_size, + access_type=root_name if root_name in {'public', 'private'} else None, + storage_type='local', + ) + ) + + unique_findings = {finding.issue_key: finding for finding in findings} + return FileReconcileScanResult( + findings=list(unique_findings.values()), + scanned_file_count=len(file_infos), + scanned_storage_count=scanned_storage_count, + ) + + @classmethod + def get_storage_roots(cls) -> dict[str, Path]: + """ + 获取对账使用的本地存储根目录 + + :return: 存储区域和绝对根目录映射 + """ + return { + 'public': Path(UploadConfig.UPLOAD_PATH).resolve(), + 'private': Path(UploadConfig.PRIVATE_UPLOAD_PATH).resolve(), + 'trash': Path(UploadConfig.FILE_TRASH_PATH).resolve(), + 'quarantine': Path(UploadConfig.FILE_RECONCILE_QUARANTINE_PATH).resolve(), + } + + @classmethod + def resolve_location(cls, root_name: str, relative_key: str) -> Path: + """ + 安全解析指定存储区域内的相对路径 + + :param root_name: 存储区域 + :param relative_key: 相对路径 + :return: 安全文件路径 + """ + roots = cls.get_storage_roots() + if root_name not in roots: + raise ValueError('存储区域不合法') + cls._validate_storage_roots(roots) + return cls._resolve_lexical_path(roots[root_name], relative_key) + + @classmethod + def calculate_file_hash(cls, filepath: Path) -> str: + """ + 计算普通文件SHA-256 + + :param filepath: 文件路径 + :return: SHA-256 + """ + return cls.calculate_file_integrity(filepath)[1] + + @classmethod + def calculate_file_integrity(cls, filepath: Path) -> tuple[int, str]: + """ + 稳定计算普通文件大小和SHA-256 + + :param filepath: 文件路径 + :return: 文件大小和SHA-256 + """ + if filepath.is_symlink() or not filepath.is_file(): + raise ValueError('目标路径不是普通文件') + before_stat = filepath.stat() + file_hasher = hashlib.sha256() + with filepath.open('rb') as file: + while chunk := file.read(cls.HASH_CHUNK_SIZE): + file_hasher.update(chunk) + after_stat = filepath.stat() + before_signature = ( + before_stat.st_dev, + before_stat.st_ino, + before_stat.st_size, + before_stat.st_mtime_ns, + ) + after_signature = ( + after_stat.st_dev, + after_stat.st_ino, + after_stat.st_size, + after_stat.st_mtime_ns, + ) + if before_signature != after_signature: + raise ValueError('文件在摘要计算期间发生变化') + return before_stat.st_size, file_hasher.hexdigest() + + @classmethod + def move_regular_file( + cls, + source_root: str, + source_key: str, + target_root: str, + target_key: str, + ) -> tuple[Path, Path]: + """ + 在受控存储区域之间移动普通文件 + + :param source_root: 来源存储区域 + :param source_key: 来源相对路径 + :param target_root: 目标存储区域 + :param target_key: 目标相对路径 + :return: 来源和目标文件路径 + """ + source_path = cls.resolve_location(source_root, source_key) + target_path = cls.resolve_location(target_root, target_key) + if source_path.is_symlink() or not source_path.is_file(): + raise ValueError('来源路径不是普通文件') + if target_path.exists() or target_path.is_symlink(): + raise FileExistsError('目标路径已存在') + target_path.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(source_path), str(target_path)) + if not target_path.is_file() or target_path.is_symlink(): + raise OSError('文件移动结果校验失败') + UploadUtil.remove_empty_directory(source_path.parent) + return source_path, target_path + + @classmethod + def delete_quarantine_file(cls, quarantine_key: str) -> Path: + """ + 永久删除隔离区普通文件 + + :param quarantine_key: 隔离区相对路径 + :return: 已删除文件路径 + """ + quarantine_path = cls.resolve_location('quarantine', quarantine_key) + if quarantine_path.is_symlink() or not quarantine_path.is_file(): + raise ValueError('隔离区目标不是普通文件') + quarantine_path.unlink() + UploadUtil.remove_empty_directory(quarantine_path.parent) + return quarantine_path + + @classmethod + def build_finding( + cls, + issue_type: str, + severity: str, + detail: str, + **metadata: Any, + ) -> FileReconcileFinding: + """ + 构造包含稳定唯一标识的对账异常 + + :return: 对账异常 + """ + identity = '|'.join( + [ + issue_type, + metadata.get('file_id') or '', + metadata.get('expected_root') or '', + metadata.get('expected_key') or '', + metadata.get('actual_root') or '', + metadata.get('actual_key') or '', + ] + ) + return FileReconcileFinding( + issue_key=hashlib.sha256(identity.encode('utf-8')).hexdigest(), + issue_type=issue_type, + severity=severity, + detail=detail, + file_id=metadata.get('file_id'), + storage_type=metadata.get('storage_type'), + access_type=metadata.get('access_type'), + expected_root=metadata.get('expected_root'), + expected_key=metadata.get('expected_key'), + actual_root=metadata.get('actual_root'), + actual_key=metadata.get('actual_key'), + expected_size=metadata.get('expected_size'), + actual_size=metadata.get('actual_size'), + expected_hash=metadata.get('expected_hash'), + actual_hash=metadata.get('actual_hash'), + ) + + @classmethod + def _build_file_context( + cls, + file_info: dict[str, Any], + roots: dict[str, Path], + ) -> _FileReconcileContext: + file_id = cls._get_text_value(file_info, 'file_id') + storage_type = cls._get_text_value(file_info, 'storage_type') + access_type = cls._get_text_value(file_info, 'access_type') + storage_key = cls._get_text_value(file_info, 'storage_key') + stored_name = cls._get_text_value(file_info, 'stored_name') + status = cls._get_text_value(file_info, 'status') + del_flag = cls._get_text_value(file_info, 'del_flag') + if not file_id: + raise ValueError('文件ID为空') + uuid.UUID(file_id) + if storage_type != 'local': + raise ValueError('暂不支持非本地存储类型') + if access_type not in {'public', 'private'}: + raise ValueError('访问类型不合法') + if not storage_key or not stored_name: + raise ValueError('存储路径或存储文件名为空') + if UploadUtil.get_original_filename(stored_name) != stored_name: + raise ValueError('存储文件名包含路径信息') + if storage_key.replace('\\', '/').rsplit('/', 1)[-1] != stored_name: + raise ValueError('存储路径和存储文件名不一致') + if (status == 'active' and del_flag != '0') or (status in {'deleted', 'purging'} and del_flag != '1'): + raise ValueError('文件状态和删除标志不一致') + if status not in {'active', 'deleted', 'purging'}: + raise ValueError('文件状态不合法') + + source_path = cls._resolve_lexical_path(roots[access_type], storage_key) + trash_key = f'{file_id}/{stored_name}' + trash_path = cls._resolve_lexical_path(roots['trash'], trash_key) + return _FileReconcileContext( + file_info=file_info, + source_root=access_type, + source_key=storage_key.replace('\\', '/'), + source_path=source_path, + trash_key=trash_key, + trash_path=trash_path, + ) + + @classmethod + def _inspect_file_context( + cls, + context: _FileReconcileContext, + roots: dict[str, Path], + expected_locations: set[tuple[str, str]], + check_hash: bool, + ) -> tuple[list[FileReconcileFinding], set[tuple[str, str]]]: + file_info = context.file_info + file_id = cls._get_text_value(file_info, 'file_id') + storage_type = cls._get_text_value(file_info, 'storage_type') + access_type = cls._get_text_value(file_info, 'access_type') + status = cls._get_text_value(file_info, 'status') + expected_size = cls._get_int_value(file_info, 'file_size') + expected_hash = cls._get_text_value(file_info, 'file_hash') + source_state = cls._get_path_state(context.source_path) + trash_state = cls._get_path_state(context.trash_path) + findings: list[FileReconcileFinding] = [] + claimed_locations: set[tuple[str, str]] = set() + expected_root = context.source_root if status == 'active' else 'trash' + expected_key = context.source_key if status == 'active' else context.trash_key + + for root_name, relative_key, path_state in ( + (context.source_root, context.source_key, source_state), + ('trash', context.trash_key, trash_state), + ): + if path_state == 'unsafe': + findings.append( + cls.build_finding( + issue_type='unsafe_entry', + severity='critical', + detail='文件记录指向符号链接或非普通文件条目', + file_id=file_id, + storage_type=storage_type, + access_type=access_type, + expected_root=expected_root, + expected_key=expected_key, + actual_root=root_name, + actual_key=relative_key, + ) + ) + if 'unsafe' in {source_state, trash_state}: + return findings, claimed_locations + + if source_state == 'file' and trash_state == 'file': + findings.append( + cls.build_finding( + issue_type='duplicate_file', + severity='critical', + detail='正式存储目录和回收区同时存在文件副本', + file_id=file_id, + storage_type=storage_type, + access_type=access_type, + expected_root=expected_root, + expected_key=expected_key, + actual_root='trash' if status == 'active' else context.source_root, + actual_key=context.trash_key if status == 'active' else context.source_key, + expected_size=expected_size, + expected_hash=expected_hash, + ) + ) + return findings, claimed_locations + + actual_path: Path | None = None + actual_root: str | None = None + actual_key: str | None = None + if status == 'active': + if source_state == 'file': + actual_path = context.source_path + actual_root = context.source_root + actual_key = context.source_key + elif trash_state == 'file': + findings.append( + cls.build_finding( + issue_type='unexpected_trash', + severity='critical', + detail='有效文件仅存在于回收区，可能由未完成的删除事务导致', + file_id=file_id, + storage_type=storage_type, + access_type=access_type, + expected_root=context.source_root, + expected_key=context.source_key, + actual_root='trash', + actual_key=context.trash_key, + expected_size=expected_size, + actual_size=context.trash_path.stat().st_size, + expected_hash=expected_hash, + ) + ) + return findings, claimed_locations + elif trash_state == 'file': + actual_path = context.trash_path + actual_root = 'trash' + actual_key = context.trash_key + elif source_state == 'file': + findings.append( + cls.build_finding( + issue_type='unexpected_source', + severity='warning', + detail='回收站文件仍位于正式存储目录，可能由未完成的恢复事务导致', + file_id=file_id, + storage_type=storage_type, + access_type=access_type, + expected_root='trash', + expected_key=context.trash_key, + actual_root=context.source_root, + actual_key=context.source_key, + expected_size=expected_size, + actual_size=context.source_path.stat().st_size, + expected_hash=expected_hash, + ) + ) + return findings, claimed_locations + + if actual_path is None: + opposite_root = 'private' if context.source_root == 'public' else 'public' + opposite_path = cls._resolve_lexical_path(roots[opposite_root], context.source_key) + opposite_location = (opposite_root, context.source_key) + if cls._get_path_state(opposite_path) == 'file' and opposite_location not in expected_locations: + claimed_locations.add(opposite_location) + findings.append( + cls.build_finding( + issue_type='wrong_storage_root', + severity='critical', + detail='文件位于与访问类型不一致的存储区域', + file_id=file_id, + storage_type=storage_type, + access_type=access_type, + expected_root=expected_root, + expected_key=expected_key, + actual_root=opposite_root, + actual_key=context.source_key, + expected_size=expected_size, + actual_size=opposite_path.stat().st_size, + expected_hash=expected_hash, + ) + ) + return findings, claimed_locations + findings.append( + cls.build_finding( + issue_type='missing_file', + severity='critical' if status == 'active' else 'warning', + detail='文件信息存在，但正式存储目录和回收区均未找到物理文件', + file_id=file_id, + storage_type=storage_type, + access_type=access_type, + expected_root=expected_root, + expected_key=expected_key, + expected_size=expected_size, + expected_hash=expected_hash, + ) + ) + return findings, claimed_locations + + findings.extend( + cls._build_integrity_findings( + file_info, + actual_path, + actual_root, + actual_key, + expected_root, + expected_key, + check_hash, + ) + ) + return findings, claimed_locations + + @classmethod + def _build_integrity_findings( + cls, + file_info: dict[str, Any], + actual_path: Path, + actual_root: str | None, + actual_key: str | None, + expected_root: str, + expected_key: str, + check_hash: bool, + ) -> list[FileReconcileFinding]: + """构造文件大小和摘要一致性异常。""" + file_id = cls._get_text_value(file_info, 'file_id') + storage_type = cls._get_text_value(file_info, 'storage_type') + access_type = cls._get_text_value(file_info, 'access_type') + expected_size = cls._get_int_value(file_info, 'file_size') + expected_hash = cls._get_text_value(file_info, 'file_hash') + actual_size = actual_path.stat().st_size + findings: list[FileReconcileFinding] = [] + if expected_size is not None and actual_size != expected_size: + findings.append( + cls.build_finding( + issue_type='size_mismatch', + severity='critical', + detail='物理文件大小与文件信息表记录不一致', + file_id=file_id, + storage_type=storage_type, + access_type=access_type, + expected_root=expected_root, + expected_key=expected_key, + actual_root=actual_root, + actual_key=actual_key, + expected_size=expected_size, + actual_size=actual_size, + expected_hash=expected_hash, + ) + ) + if check_hash and expected_hash: + actual_hash = cls.calculate_file_hash(actual_path) + if actual_hash != expected_hash: + findings.append( + cls.build_finding( + issue_type='hash_mismatch', + severity='critical', + detail='物理文件SHA-256与文件信息表记录不一致', + file_id=file_id, + storage_type=storage_type, + access_type=access_type, + expected_root=expected_root, + expected_key=expected_key, + actual_root=actual_root, + actual_key=actual_key, + expected_size=expected_size, + actual_size=actual_size, + expected_hash=expected_hash, + actual_hash=actual_hash, + ) + ) + return findings + + @classmethod + def _iter_storage_entries(cls, root: Path) -> Iterator[tuple[str, Path, bool]]: + """逐项遍历存储目录，避免大目录扫描时一次性占用过多内存。""" + stack = [root] + while stack: + directory = stack.pop() + with os.scandir(directory) as iterator: + for entry in iterator: + entry_path = Path(entry.path) + relative_key = entry_path.relative_to(root).as_posix() + if entry.is_symlink(): + yield relative_key, entry_path, True + elif entry.is_dir(follow_symlinks=False): + stack.append(entry_path) + elif entry.is_file(follow_symlinks=False): + yield relative_key, entry_path, False + else: + yield relative_key, entry_path, True + + @classmethod + def _resolve_lexical_path(cls, root: Path, relative_key: str) -> Path: + FilePathUtil.resolve_path_within_root(root, relative_key) + normalized_key = relative_key.replace('\\', '/') + candidate_path = root.joinpath(*normalized_key.split('/')) + current_path = root + for part in normalized_key.split('/'): + current_path = current_path / part + if current_path.is_symlink(): + raise ValueError('存储路径包含符号链接') + return candidate_path + + @classmethod + def _validate_storage_roots(cls, roots: dict[str, Path]) -> None: + root_items = list(roots.items()) + for index, (root_name, root_path) in enumerate(root_items): + root_path.mkdir(parents=True, exist_ok=True) + for other_name, other_path in root_items[index + 1 :]: + if ( + root_path == other_path + or cls._is_relative_to(root_path, other_path) + or cls._is_relative_to(other_path, root_path) + ): + raise ValueError(f'存储区域{root_name}和{other_name}不能相同或相互嵌套') + + @staticmethod + def _is_relative_to(path: Path, parent: Path) -> bool: + try: + path.relative_to(parent) + return True + except ValueError: + return False + + @staticmethod + def _get_path_state(path: Path) -> str: + if path.is_symlink(): + return 'unsafe' + if not path.exists(): + return 'missing' + return 'file' if path.is_file() else 'unsafe' + + @staticmethod + def _get_text_value(file_info: dict[str, Any], field_name: str) -> str | None: + value = file_info.get(field_name) + return str(value) if value is not None else None + + @staticmethod + def _get_int_value(file_info: dict[str, Any], field_name: str) -> int | None: + value = file_info.get(field_name) + return int(value) if value is not None else None diff --git a/ruoyi-fastapi-backend/utils/response_util.py b/ruoyi-fastapi-backend/utils/response_util.py index d281501..2208ecd 100644 --- a/ruoyi-fastapi-backend/utils/response_util.py +++ b/ruoyi-fastapi-backend/utils/response_util.py @@ -300,6 +300,7 @@ class ResponseUtil: headers: Mapping[str, str] | None = None, media_type: str | None = None, background: BackgroundTask | None = None, + status_code: int = status.HTTP_200_OK, ) -> Response: """ 流式响应方法 @@ -308,8 +309,13 @@ class ResponseUtil: :param headers: 可选，响应头信息 :param media_type: 可选，响应结果媒体类型 :param background: 可选，响应返回后执行的后台任务 + :param status_code: 响应状态码 :return: 流式响应结果 """ return StreamingResponse( - status_code=status.HTTP_200_OK, content=data, headers=headers, media_type=media_type, background=background + status_code=status_code, + content=data, + headers=headers, + media_type=media_type, + background=background, ) diff --git a/ruoyi-fastapi-backend/utils/upload_util.py b/ruoyi-fastapi-backend/utils/upload_util.py index b751dde..1a8806e 100644 --- a/ruoyi-fastapi-backend/utils/upload_util.py +++ b/ruoyi-fastapi-backend/utils/upload_util.py @@ -1,19 +1,84 @@ import os import random +import re from collections.abc import AsyncGenerator from datetime import datetime +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import TYPE_CHECKING +from urllib.parse import quote import aiofiles from fastapi import UploadFile from config.env import UploadConfig +if TYPE_CHECKING: + from utils.file_util import FileByteRange + + +class FilePathUtil: + """ + 文件路径安全校验工具类 + """ + + @classmethod + def resolve_path_within_root(cls, root: str | os.PathLike[str], untrusted_path: str) -> Path: + """ + 将不可信相对路径解析为根目录内路径 + + :param root: 文件根目录 + :param untrusted_path: 不可信文件路径 + :return: 根目录内的路径 + """ + if not untrusted_path or '\x00' in untrusted_path: + raise ValueError('文件路径不能为空') + + normalized_path = untrusted_path.replace('\\', '/') + windows_path = PureWindowsPath(untrusted_path) + posix_path = PurePosixPath(normalized_path) + path_parts = normalized_path.split('/') + + if ( + windows_path.is_absolute() + or bool(windows_path.drive) + or posix_path.is_absolute() + or any(part in {'', '.', '..'} for part in path_parts) + ): + raise ValueError('文件路径不合法') + + root_path = Path(root).resolve() + candidate_path = root_path.joinpath(*path_parts).resolve() + try: + candidate_path.relative_to(root_path) + except ValueError as exc: + raise ValueError('文件路径超出允许目录') from exc + + return candidate_path + + @classmethod + def resolve_file_within_root(cls, root: str | os.PathLike[str], untrusted_path: str) -> Path: + """ + 将不可信相对路径解析为根目录内已存在的普通文件 + + :param root: 文件根目录 + :param untrusted_path: 不可信文件路径 + :return: 根目录内的文件路径 + """ + candidate_path = cls.resolve_path_within_root(root, untrusted_path) + + if not candidate_path.is_file(): + raise FileNotFoundError('文件不存在') + return candidate_path + class UploadUtil: """ 上传工具类 """ + GENERATED_FILE_INFO_LENGTH = 18 + MAX_RANDOM_CODE = 999 + @classmethod def generate_random_number(cls) -> str: """ @@ -26,7 +91,7 @@ class UploadUtil: return f'{random_number:03}' @classmethod - def check_file_exists(cls, filepath: str) -> bool: + def check_file_exists(cls, filepath: str | os.PathLike[str]) -> bool: """ 检查文件是否存在 @@ -35,6 +100,15 @@ class UploadUtil: """ return os.path.exists(filepath) + @classmethod + def ensure_directory(cls, directory: str | os.PathLike[str]) -> None: + """ + 创建文件目录 + + :param directory: 文件目录 + """ + os.makedirs(directory, exist_ok=True) + @classmethod def check_file_extension(cls, file: UploadFile) -> bool: """ @@ -43,10 +117,84 @@ class UploadUtil: :param file: 文件对象 :return: 校验结果 """ - file_extension = file.filename.rsplit('.', 1)[-1] + file_extension = cls.get_file_extension(file.filename) return file_extension in UploadConfig.DEFAULT_ALLOWED_EXTENSION + @classmethod + def get_file_extension(cls, filename: str | None) -> str: + """ + 获取文件名的小写扩展名 + + :param filename: 文件名称 + :return: 小写文件扩展名 + """ + if not filename: + return '' + safe_name = PurePosixPath(filename.replace('\\', '/')).name + return Path(safe_name).suffix.lower().removeprefix('.') + + @classmethod + def get_original_filename(cls, filename: str | None) -> str: + """ + 获取移除目录信息后的原始文件名 + + :param filename: 文件名称 + :return: 原始文件名 + """ + if not filename: + return '' + return PurePosixPath(filename.replace('\\', '/')).name + + @classmethod + def get_safe_file_stem(cls, filename: str | None) -> str: + """ + 获取移除路径和非法字符后的文件名前缀 + + :param filename: 文件名称 + :return: 安全文件名前缀 + """ + original_filename = cls.get_original_filename(filename) + file_stem = original_filename.rsplit('.', 1)[0] + safe_file_stem = re.sub(r'[<>:"/\\|?*\x00-\x1f]', '_', file_stem) + while '..' in safe_file_stem: + safe_file_stem = safe_file_stem.replace('..', '_') + safe_file_stem = safe_file_stem.strip(' ._')[:100] + return safe_file_stem or 'file' + + @classmethod + def build_download_headers( + cls, + filename: str, + byte_range: 'FileByteRange | None' = None, + accept_ranges: bool = True, + ) -> dict[str, str]: + """ + 构造文件下载响应头 + + :param filename: 文件名称 + :param byte_range: 文件字节范围 + :param accept_ranges: 是否支持Range请求 + :return: 文件下载响应头 + """ + safe_name = cls.get_original_filename(filename) or 'download' + encoded_name = quote(safe_name) + headers = { + 'Content-Disposition': f"attachment; filename*=UTF-8''{encoded_name}", + 'download-filename': encoded_name, + 'Accept-Ranges': 'bytes' if accept_ranges else 'none', + 'X-Content-Type-Options': 'nosniff', + 'Content-Security-Policy': ( + "sandbox; default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" + ), + 'X-Frame-Options': 'DENY', + } + if byte_range is not None: + headers['Content-Length'] = str(byte_range.length) + if byte_range.is_partial: + headers['Content-Range'] = f'bytes {byte_range.start}-{byte_range.end}/{byte_range.file_size}' + return headers + @classmethod def check_file_timestamp(cls, filename: str) -> bool: """ @@ -55,9 +203,10 @@ class UploadUtil: :param filename: 文件名称 :return: 校验结果 """ - timestamp = ( - filename.rsplit('.', 1)[0].rsplit('_', maxsplit=1)[-1].split(UploadConfig.UPLOAD_MACHINE, maxsplit=1)[0] - ) + generated_file_info = filename.rsplit('.', 1)[0].rsplit('_', maxsplit=1)[-1] + if len(generated_file_info) != cls.GENERATED_FILE_INFO_LENGTH: + return False + timestamp = generated_file_info[:14] try: datetime.strptime(timestamp, '%Y%m%d%H%M%S') return True @@ -72,7 +221,11 @@ class UploadUtil: :param filename: 文件名称 :return: 校验结果 """ - return filename.rsplit('.', 1)[0][-4] == UploadConfig.UPLOAD_MACHINE + generated_file_info = filename.rsplit('.', 1)[0].rsplit('_', maxsplit=1)[-1] + return ( + len(generated_file_info) == cls.GENERATED_FILE_INFO_LENGTH + and generated_file_info[-4] == UploadConfig.UPLOAD_MACHINE + ) @classmethod def check_file_random_code(cls, filename: str) -> bool: @@ -82,27 +235,71 @@ class UploadUtil: :param filename: 文件名称 :return: 校验结果 """ - valid_code_list = [f'{i:03}' for i in range(1, 999)] - - return filename.rsplit('.', 1)[0][-3:] in valid_code_list + generated_file_info = filename.rsplit('.', 1)[0].rsplit('_', maxsplit=1)[-1] + random_code = generated_file_info[-3:] + return ( + len(generated_file_info) == cls.GENERATED_FILE_INFO_LENGTH + and random_code.isdigit() + and 1 <= int(random_code) <= cls.MAX_RANDOM_CODE + ) @classmethod - async def generate_file(cls, filepath: str) -> AsyncGenerator[bytes, None]: + async def generate_file( + cls, + filepath: str | os.PathLike[str], + start: int = 0, + length: int | None = None, + ) -> AsyncGenerator[bytes, None]: """ 根据文件生成二进制数据 :param filepath: 文件路径 + :param start: 读取起始字节位置 + :param length: 读取字节数 :yield: 二进制数据 """ + if start < 0 or (length is not None and length < 0): + raise ValueError('文件读取范围不合法') async with aiofiles.open(filepath, 'rb') as response_file: - async for chunk in response_file: + await response_file.seek(start) + remaining = length + while remaining is None or remaining > 0: + chunk_size = 1024 * 1024 if remaining is None else min(1024 * 1024, remaining) + chunk = await response_file.read(chunk_size) + if not chunk: + break + if remaining is not None: + remaining -= len(chunk) yield chunk @classmethod - def delete_file(cls, filepath: str) -> None: + def delete_file(cls, filepath: str | os.PathLike[str]) -> None: """ 根据文件路径删除对应文件 :param filepath: 文件路径 """ os.remove(filepath) + + @classmethod + def move_file(cls, source: str | os.PathLike[str], target: str | os.PathLike[str]) -> None: + """ + 移动文件到目标路径 + + :param source: 原文件路径 + :param target: 目标文件路径 + """ + cls.ensure_directory(Path(target).parent) + os.replace(source, target) + + @classmethod + def remove_empty_directory(cls, directory: str | os.PathLike[str]) -> None: + """ + 删除空目录 + + :param directory: 目录路径 + """ + try: + os.rmdir(directory) + except OSError: + pass diff --git a/ruoyi-fastapi-frontend/src/api/system/file.js b/ruoyi-fastapi-frontend/src/api/system/file.js new file mode 100644 index 0000000..af84023 --- /dev/null +++ b/ruoyi-fastapi-frontend/src/api/system/file.js @@ -0,0 +1,241 @@ +import request from '@/utils/request' + +// 查询文件列表 +export function listFile(query) { + return request({ + url: '/system/file/list', + method: 'get', + params: query + }) +} + +// 查询文件统计 +export function getFileStats(query) { + return request({ + url: '/system/file/stats', + method: 'get', + params: query + }) +} + +// 查询文件存储对账异常 +export function listFileReconcileIssue(query) { + return request({ + url: '/system/file/reconcile/issues/list', + method: 'get', + params: query + }) +} + +// 查询文件存储对账任务 +export function listFileReconcileRun(query) { + return request({ + url: '/system/file/reconcile/runs/list', + method: 'get', + params: query + }) +} + +// 查询文件存储对账统计 +export function getFileReconcileStats() { + return request({ + url: '/system/file/reconcile/stats', + method: 'get' + }) +} + +// 启动文件存储对账任务 +export function startFileReconcile(data) { + return request({ + url: '/system/file/reconcile/run', + method: 'post', + data: data + }) +} + +// 处理文件存储对账异常 +export function handleFileReconcileIssue(issueId, data) { + return request({ + url: '/system/file/reconcile/issues/' + issueId, + method: 'put', + data: data + }) +} + +// 查询文件业务保留策略 +export function listFileRetentionPolicy() { + return request({ + url: '/system/file/retention-policy/list', + method: 'get' + }) +} + +// 新增文件业务保留策略 +export function addFileRetentionPolicy(data) { + return request({ + url: '/system/file/retention-policy', + method: 'post', + data: data + }) +} + +// 修改文件业务保留策略 +export function updateFileRetentionPolicy(data) { + return request({ + url: '/system/file/retention-policy', + method: 'put', + data: data + }) +} + +// 删除文件业务保留策略 +export function delFileRetentionPolicy(businessType) { + return request({ + url: '/system/file/retention-policy/' + encodeURIComponent(businessType), + method: 'delete' + }) +} + +// 查询文件保留期限提醒 +export function listFileRetentionReminder(query) { + return request({ + url: '/system/file/retention-reminder/list', + method: 'get', + params: query + }) +} + +// 执行文件保留期限提醒扫描 +export function scanFileRetentionReminder() { + return request({ + url: '/system/file/retention-reminder/scan', + method: 'post' + }) +} + +// 标记文件保留期限提醒为已读 +export function readFileRetentionReminder(noticeIds) { + return request({ + url: '/system/file/retention-reminder/' + noticeIds + '/read', + method: 'put' + }) +} + +// 延长文件保留期限 +export function extendFileRetention(noticeId, data) { + return request({ + url: '/system/file/retention-reminder/' + noticeId + '/extend', + method: 'put', + data: data + }) +} + +// 处置到期文件 +export function disposeExpiredFile(noticeId, data) { + return request({ + url: '/system/file/retention-reminder/' + noticeId + '/dispose', + method: 'put', + data: data + }) +} + +// 查询文件详情 +export function getFile(fileId) { + return request({ + url: '/system/file/' + fileId, + method: 'get' + }) +} + +// 查询文件业务引用列表 +export function listFileReference(fileId) { + return request({ + url: '/system/file/' + fileId + '/reference/list', + method: 'get' + }) +} + +// 查询文件访问审计列表 +export function listFileAccessLog(fileId, query) { + return request({ + url: '/system/file/' + fileId + '/access-log/list', + method: 'get', + params: query + }) +} + +// 查询文件访问控制列表 +export function listFileAcl(fileId) { + return request({ + url: '/system/file/' + fileId + '/acl/list', + method: 'get' + }) +} + +// 查询文件授权主体选项 +export function searchFileAclSubjects(query) { + return request({ + url: '/system/file/acl/subjects', + method: 'get', + params: query + }) +} + +// 查询文件授权部门树 +export function getFileAclDeptTree() { + return request({ + url: '/system/file/acl/dept-tree', + method: 'get' + }) +} + +// 保存文件访问控制配置 +export function saveFileAcl(fileId, data) { + return request({ + url: '/system/file/' + fileId + '/acl', + method: 'put', + data: data + }) +} + +// 批量保存文件访问控制配置 +export function batchSaveFileAcl(data) { + return request({ + url: '/system/file/acl/batch', + method: 'put', + data: data + }) +} + +// 转移文件所有者和所属部门 +export function transferFile(fileIds, data) { + return request({ + url: '/system/file/' + fileIds + '/transfer', + method: 'put', + data: data + }) +} + +// 恢复文件 +export function restoreFile(fileIds) { + return request({ + url: '/system/file/' + fileIds + '/restore', + method: 'put' + }) +} + +// 永久清理回收站文件 +export function purgeFile(fileIds) { + return request({ + url: '/system/file/purge/' + fileIds, + method: 'delete' + }) +} + +// 将文件移入回收站 +export function delFile(fileIds) { + return request({ + url: '/system/file/' + fileIds, + method: 'delete' + }) +} diff --git a/ruoyi-fastapi-frontend/src/components/BusinessFileUpload/index.vue b/ruoyi-fastapi-frontend/src/components/BusinessFileUpload/index.vue new file mode 100644 index 0000000..85c2347 --- /dev/null +++ b/ruoyi-fastapi-frontend/src/components/BusinessFileUpload/index.vue @@ -0,0 +1,565 @@ + + + + + diff --git a/ruoyi-fastapi-frontend/src/components/FileUpload/index.vue b/ruoyi-fastapi-frontend/src/components/FileUpload/index.vue index 0cfe4dc..7ff6961 100644 --- a/ruoyi-fastapi-frontend/src/components/FileUpload/index.vue +++ b/ruoyi-fastapi-frontend/src/components/FileUpload/index.vue @@ -29,7 +29,12 @@
  • - + {{ getFileName(file.name) }}
    @@ -51,6 +56,11 @@ const props = defineProps({ type: String, default: "/common/upload" }, + // 是否上传受保护文件 + isPrivate: { + type: Boolean, + default: false + }, // 上传携带的参数 data: { type: Object @@ -92,7 +102,8 @@ const emit = defineEmits(); const number = ref(0); const uploadList = ref([]); const baseUrl = import.meta.env.VITE_APP_BASE_API; -const uploadFileUrl = ref(import.meta.env.VITE_APP_BASE_API + props.action); // 上传文件服务器地址 +const uploadAction = props.isPrivate && props.action === "/common/upload" ? "/common/files/upload" : props.action; +const uploadFileUrl = ref(import.meta.env.VITE_APP_BASE_API + uploadAction); // 上传文件服务器地址 const headers = ref({ Authorization: "Bearer " + getToken() }); const fileList = ref([]); const showTip = computed( @@ -173,6 +184,14 @@ function handleUploadSuccess(res, file) { } } +// 下载受保护文件 +function handleFileDownload(event, file) { + if (typeof file.url === "string" && file.url.startsWith("/common/files/")) { + event.preventDefault(); + proxy.$download.file(file.url); + } +} + // 删除文件 function handleDelete(index) { fileList.value.splice(index, 1); diff --git a/ruoyi-fastapi-frontend/src/main.js b/ruoyi-fastapi-frontend/src/main.js index f1f6fe2..9bd467b 100644 --- a/ruoyi-fastapi-frontend/src/main.js +++ b/ruoyi-fastapi-frontend/src/main.js @@ -37,6 +37,8 @@ import RightToolbar from '@/components/RightToolbar' import Editor from "@/components/Editor" // 文件上传组件 import FileUpload from "@/components/FileUpload" +// 业务附件上传组件 +import BusinessFileUpload from "@/components/BusinessFileUpload" // 图片上传组件 import ImageUpload from "@/components/ImageUpload" // 图片预览组件 @@ -61,6 +63,7 @@ app.config.globalProperties.selectDictLabels = selectDictLabels app.component('DictTag', DictTag) app.component('Pagination', Pagination) app.component('FileUpload', FileUpload) +app.component('BusinessFileUpload', BusinessFileUpload) app.component('ImageUpload', ImageUpload) app.component('ImagePreview', ImagePreview) app.component('RightToolbar', RightToolbar) diff --git a/ruoyi-fastapi-frontend/src/plugins/download.js b/ruoyi-fastapi-frontend/src/plugins/download.js index 1a89efd..34520e8 100644 --- a/ruoyi-fastapi-frontend/src/plugins/download.js +++ b/ruoyi-fastapi-frontend/src/plugins/download.js @@ -6,8 +6,107 @@ import errorCode from '@/utils/errorCode' import { blobValidate } from '@/utils/ruoyi' const baseURL = import.meta.env.VITE_APP_BASE_API +const DOWNLOAD_CHUNK_SIZE = 8 * 1024 * 1024 +const DOWNLOAD_CHUNK_RETRY_COUNT = 2 let downloadLoadingInstance; +function parseContentRange(contentRange) { + const rangeMatch = /^bytes (\d+)-(\d+)\/(\d+)$/i.exec(contentRange || '') + if (!rangeMatch) { + throw new Error('分段下载响应缺少有效的Content-Range') + } + return { + start: Number(rangeMatch[1]), + end: Number(rangeMatch[2]), + total: Number(rangeMatch[3]) + } +} + +function isRetryableDownloadError(error) { + return !error.response || error.response.status >= 500 +} + +async function requestDownloadChunk(url, start) { + let lastError + for (let retryCount = 0; retryCount <= DOWNLOAD_CHUNK_RETRY_COUNT; retryCount++) { + try { + return await axios({ + method: 'get', + url, + responseType: 'blob', + headers: { + 'Authorization': 'Bearer ' + getToken(), + 'Range': `bytes=${start}-${start + DOWNLOAD_CHUNK_SIZE - 1}` + } + }) + } catch (error) { + lastError = error + if (!isRetryableDownloadError(error) || retryCount === DOWNLOAD_CHUNK_RETRY_COUNT) { + throw error + } + } + } + throw lastError +} + +function requestFullDownload(url) { + return axios({ + method: 'get', + url, + responseType: 'blob', + headers: { 'Authorization': 'Bearer ' + getToken() } + }) +} + +async function downloadByRange(url) { + const chunks = [] + let filename + let contentType + let nextStart = 0 + + while (true) { + let response + try { + response = await requestDownloadChunk(url, nextStart) + } catch (error) { + if (nextStart !== 0 || error.response?.status !== 416) { + throw error + } + response = await requestFullDownload(url) + } + if (!blobValidate(response.data)) { + return { errorData: response.data } + } + + filename = filename || response.headers['download-filename'] + contentType = contentType || response.data.type + if (response.status !== 206) { + return { + blob: new Blob([response.data], { type: contentType }), + filename + } + } + + const contentRange = parseContentRange(response.headers['content-range']) + if ( + contentRange.start !== nextStart || + contentRange.end < contentRange.start || + contentRange.total <= contentRange.end || + response.data.size !== contentRange.end - contentRange.start + 1 + ) { + throw new Error('分段下载响应范围不一致') + } + chunks.push(response.data) + if (contentRange.end + 1 === contentRange.total) { + return { + blob: new Blob(chunks, { type: contentType }), + filename + } + } + nextStart = contentRange.end + 1 + } +} + export default { name(name, isDelete = true) { var url = baseURL + "/common/download?fileName=" + encodeURIComponent(name) + "&delete=" + isDelete @@ -26,22 +125,34 @@ export default { } }) }, - resource(resource) { + async resource(resource) { var url = baseURL + "/common/download/resource?resource=" + encodeURIComponent(resource); - axios({ - method: 'get', - url: url, - responseType: 'blob', - headers: { 'Authorization': 'Bearer ' + getToken() } - }).then((res) => { - const isBlob = blobValidate(res.data); - if (isBlob) { - const blob = new Blob([res.data]) - this.saveAs(blob, decodeURIComponent(res.headers['download-filename'])) - } else { - this.printErrMsg(res.data); + try { + const result = await downloadByRange(url) + if (result.errorData) { + await this.printErrMsg(result.errorData) + return } - }) + this.saveAs(result.blob, decodeURIComponent(result.filename)) + } catch (error) { + console.error(error) + ElMessage.error('下载文件出现错误，请联系管理员！') + } + }, + async file(resource) { + var url = baseURL + resource; + try { + const result = await downloadByRange(url) + if (result.errorData) { + await this.printErrMsg(result.errorData) + return + } + const fallbackFileName = resource.split('/').pop(); + this.saveAs(result.blob, result.filename ? decodeURIComponent(result.filename) : fallbackFileName) + } catch (error) { + console.error(error) + ElMessage.error('下载文件出现错误，请联系管理员！') + } }, zip(url, name) { var url = baseURL + url diff --git a/ruoyi-fastapi-frontend/src/utils/transportCryptoPolicy.js b/ruoyi-fastapi-frontend/src/utils/transportCryptoPolicy.js index b0b3783..d85cc5c 100644 --- a/ruoyi-fastapi-frontend/src/utils/transportCryptoPolicy.js +++ b/ruoyi-fastapi-frontend/src/utils/transportCryptoPolicy.js @@ -7,7 +7,9 @@ const EXCLUDED_URL_PATTERNS = [ '/transport/crypto/frontend-config', '/transport/crypto/public-key', '/common/download', - '/common/download/resource' + '/common/download/resource', + '/common/files/', + '/system/file/download/' ] const TRANSPORT_FRONTEND_CONFIG_CACHE_KEY = 'transportCryptoFrontendConfig' const TRANSPORT_FRONTEND_CONFIG_URL = '/transport/crypto/frontend-config' diff --git a/ruoyi-fastapi-frontend/src/views/system/file/components/FileAclDrawer.vue b/ruoyi-fastapi-frontend/src/views/system/file/components/FileAclDrawer.vue new file mode 100644 index 0000000..930ed07 --- /dev/null +++ b/ruoyi-fastapi-frontend/src/views/system/file/components/FileAclDrawer.vue @@ -0,0 +1,383 @@ + + + + + diff --git a/ruoyi-fastapi-frontend/src/views/system/file/components/FileAuditDrawer.vue b/ruoyi-fastapi-frontend/src/views/system/file/components/FileAuditDrawer.vue new file mode 100644 index 0000000..cc56a13 --- /dev/null +++ b/ruoyi-fastapi-frontend/src/views/system/file/components/FileAuditDrawer.vue @@ -0,0 +1,303 @@ + + + + + diff --git a/ruoyi-fastapi-frontend/src/views/system/file/components/FileDetailDialog.vue b/ruoyi-fastapi-frontend/src/views/system/file/components/FileDetailDialog.vue new file mode 100644 index 0000000..c91d35e --- /dev/null +++ b/ruoyi-fastapi-frontend/src/views/system/file/components/FileDetailDialog.vue @@ -0,0 +1,170 @@ + + + + + diff --git a/ruoyi-fastapi-frontend/src/views/system/file/components/FileReconcileDrawer.vue b/ruoyi-fastapi-frontend/src/views/system/file/components/FileReconcileDrawer.vue new file mode 100644 index 0000000..501dc83 --- /dev/null +++ b/ruoyi-fastapi-frontend/src/views/system/file/components/FileReconcileDrawer.vue @@ -0,0 +1,770 @@ + + + + + diff --git a/ruoyi-fastapi-frontend/src/views/system/file/components/FileReferenceDrawer.vue b/ruoyi-fastapi-frontend/src/views/system/file/components/FileReferenceDrawer.vue new file mode 100644 index 0000000..43e5289 --- /dev/null +++ b/ruoyi-fastapi-frontend/src/views/system/file/components/FileReferenceDrawer.vue @@ -0,0 +1,106 @@ + + + diff --git a/ruoyi-fastapi-frontend/src/views/system/file/components/FileRetentionPolicyDrawer.vue b/ruoyi-fastapi-frontend/src/views/system/file/components/FileRetentionPolicyDrawer.vue new file mode 100644 index 0000000..007e52b --- /dev/null +++ b/ruoyi-fastapi-frontend/src/views/system/file/components/FileRetentionPolicyDrawer.vue @@ -0,0 +1,248 @@ + + + diff --git a/ruoyi-fastapi-frontend/src/views/system/file/components/FileRetentionReminderDrawer.vue b/ruoyi-fastapi-frontend/src/views/system/file/components/FileRetentionReminderDrawer.vue new file mode 100644 index 0000000..bda2989 --- /dev/null +++ b/ruoyi-fastapi-frontend/src/views/system/file/components/FileRetentionReminderDrawer.vue @@ -0,0 +1,437 @@ + + + diff --git a/ruoyi-fastapi-frontend/src/views/system/file/components/FileSearchForm.vue b/ruoyi-fastapi-frontend/src/views/system/file/components/FileSearchForm.vue new file mode 100644 index 0000000..f5138ba --- /dev/null +++ b/ruoyi-fastapi-frontend/src/views/system/file/components/FileSearchForm.vue @@ -0,0 +1,132 @@ + + + diff --git a/ruoyi-fastapi-frontend/src/views/system/file/components/FileStatistics.vue b/ruoyi-fastapi-frontend/src/views/system/file/components/FileStatistics.vue new file mode 100644 index 0000000..c5b9515 --- /dev/null +++ b/ruoyi-fastapi-frontend/src/views/system/file/components/FileStatistics.vue @@ -0,0 +1,272 @@ + + + + + diff --git a/ruoyi-fastapi-frontend/src/views/system/file/components/FileTable.vue b/ruoyi-fastapi-frontend/src/views/system/file/components/FileTable.vue new file mode 100644 index 0000000..6515cb7 --- /dev/null +++ b/ruoyi-fastapi-frontend/src/views/system/file/components/FileTable.vue @@ -0,0 +1,297 @@ + + + + + diff --git a/ruoyi-fastapi-frontend/src/views/system/file/components/FileTransferDialog.vue b/ruoyi-fastapi-frontend/src/views/system/file/components/FileTransferDialog.vue new file mode 100644 index 0000000..eea96ea --- /dev/null +++ b/ruoyi-fastapi-frontend/src/views/system/file/components/FileTransferDialog.vue @@ -0,0 +1,175 @@ + + + + + diff --git a/ruoyi-fastapi-frontend/src/views/system/file/components/fileFormatters.js b/ruoyi-fastapi-frontend/src/views/system/file/components/fileFormatters.js new file mode 100644 index 0000000..519f7c6 --- /dev/null +++ b/ruoyi-fastapi-frontend/src/views/system/file/components/fileFormatters.js @@ -0,0 +1,146 @@ +import { parseTime } from "@/utils/ruoyi"; + +export function formatFileSize(size) { + const bytes = Number(size || 0); + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(2)} KB`; + if (bytes < 1024 * 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(2)} MB`; + return `${(bytes / 1024 / 1024 / 1024).toFixed(2)} GB`; +} + +export function accessTypeLabel(accessType) { + return accessType === "public" ? "公开文件" : "受保护文件"; +} + +export function statusLabel(status) { + return { + active: "正常", + deleted: "已删除", + purging: "清理中" + }[status] || status; +} + +export function expirationLabel(expireTime) { + if (!expireTime) return "永久有效"; + const remainingTime = new Date(expireTime).getTime() - Date.now(); + if (remainingTime <= 0) return "已过期"; + if (remainingTime <= 7 * 24 * 60 * 60 * 1000) return "即将过期"; + return parseTime(expireTime, "{y}-{m}-{d}"); +} + +export function expirationTagType(expireTime) { + if (!expireTime) return "info"; + const remainingTime = new Date(expireTime).getTime() - Date.now(); + if (remainingTime <= 0) return "danger"; + return remainingTime <= 7 * 24 * 60 * 60 * 1000 ? "warning" : "success"; +} + +export function isAclExpiring(expireTime) { + if (!expireTime) return false; + const remainingTime = new Date(expireTime).getTime() - Date.now(); + return remainingTime > 0 && remainingTime <= 7 * 24 * 60 * 60 * 1000; +} + +export function storageStatusLabel(storageStatus) { + return { + normal: "正常", + missing: "文件缺失", + quarantined: "已隔离", + invalid: "异常" + }[storageStatus] || "未知"; +} + +export function storageStatusTagType(storageStatus) { + return { + normal: "success", + missing: "danger", + quarantined: "warning", + invalid: "danger" + }[storageStatus] || "info"; +} + +export function actionLabel(action) { + return { + upload: "上传", + download: "下载", + acl_update: "授权变更", + transfer: "归属转移", + delete: "移入回收站", + restore: "恢复", + purge: "永久清理", + reconcile: "存储对账", + retention_extend: "保留延期", + retention_dispose: "到期处置" + }[action] || action; +} + +export function resultLabel(result) { + return { + allowed: "已授权", + denied: "已拒绝", + completed: "已完成", + failed: "失败" + }[result] || result; +} + +export function resultTagType(result) { + return { + allowed: "primary", + denied: "danger", + completed: "success", + failed: "danger" + }[result] || "info"; +} + +export function parseOperationDetail(operationDetail) { + if (!operationDetail) return []; + try { + const detail = JSON.parse(operationDetail); + return Object.entries(detail).map(([key, value]) => ({ + label: auditDetailKeyLabel(key), + value: typeof value === "object" ? JSON.stringify(value, null, 2) : String(value ?? "-") + })); + } catch { + return [{ label: "操作详情", value: operationDetail }]; + } +} + +function auditDetailKeyLabel(key) { + return { + previousAclVersion: "原权限版本", + newAclVersion: "新权限版本", + entryCount: "授权项数量", + allowCount: "允许项数量", + denyCount: "拒绝项数量", + subjectTypeCounts: "主体类型统计", + previousOwnerUserId: "原所有者ID", + previousDeptId: "原所属部门ID", + newOwnerUserId: "新所有者ID", + newOwnerName: "新所有者", + newDeptId: "新所属部门ID", + reason: "操作原因", + originalName: "原始文件名", + accessType: "访问类型", + referenceCount: "业务引用数量", + previousStatus: "原状态", + newStatus: "新状态", + truncated: "详情已截断", + preview: "详情预览", + issueId: "对账异常ID", + issueType: "异常类型", + action: "处理动作", + actualRoot: "实际存储区域", + actualKey: "实际相对路径", + expectedRoot: "预期存储区域", + expectedKey: "预期相对路径", + previousExpireTime: "原到期时间", + newExpireTime: "新到期时间", + expireTime: "到期时间", + range: "请求范围", + rangeStart: "分段起始字节", + rangeEnd: "分段结束字节", + fileSize: "文件总大小", + releasedReferenceCount: "解除引用数量", + releasedReferences: "解除引用明细" + }[key] || key; +} diff --git a/ruoyi-fastapi-frontend/src/views/system/file/index.vue b/ruoyi-fastapi-frontend/src/views/system/file/index.vue new file mode 100644 index 0000000..f427a7f --- /dev/null +++ b/ruoyi-fastapi-frontend/src/views/system/file/index.vue @@ -0,0 +1,375 @@ + + +