diff --git a/backend/conf/env.py b/backend/conf/env.py index d71cdf6..f22d39d 100644 --- a/backend/conf/env.py +++ b/backend/conf/env.py @@ -67,3 +67,14 @@ EMAIL_PORT = 25 # 服务器不支持STARTTLS,使用纯明文连接 EMAIL_USE_SSL = False EMAIL_USE_TLS = False + +# ================================================= # +# *************** RustFS 对象存储 配置 *************** # +# ================================================= # +RUSTFS_ENDPOINT = "192.168.80.90:9000" +RUSTFS_ACCESS_KEY = "kqaQFK3BMETpd2TcWBnT" +RUSTFS_SECRET_KEY = "39Ms49eTkMW9E633AjFLcnuyFbgt2H1du4nLzpZe" +RUSTFS_BUCKET = "pis-media" +RUSTFS_SECURE = False # True=HTTPS, False=HTTP +RUSTFS_PATH_PREFIX = "" # 存储路径前缀,可为空 +RUSTFS_REGION = "us-east-1" # RustFS 区域(可自定义) diff --git a/backend/dvadmin/system/views/file_list.py b/backend/dvadmin/system/views/file_list.py index bc58e17..8a43495 100644 --- a/backend/dvadmin/system/views/file_list.py +++ b/backend/dvadmin/system/views/file_list.py @@ -36,8 +36,6 @@ class FileSerializer(CustomModelSerializer): fields = "__all__" def create(self, validated_data): - file_engine = dispatch.get_system_config_values("file_storage.file_engine") or 'local' - file_backup = dispatch.get_system_config_values("file_storage.file_backup") file = self.initial_data.get('file') file_size = file.size validated_data['name'] = str(file) @@ -46,28 +44,19 @@ class FileSerializer(CustomModelSerializer): for chunk in file.chunks(): md5.update(chunk) validated_data['md5sum'] = md5.hexdigest() - validated_data['engine'] = file_engine + validated_data['engine'] = 'rustfs' validated_data['mime_type'] = file.content_type ft = {'image':0,'video':1,'audio':2}.get(file.content_type.split('/')[0], None) validated_data['file_type'] = 3 if ft is None else ft - if file_backup: - validated_data['url'] = file - if file_engine == 'oss': - from dvadmin.utils.aliyunoss import ali_oss_upload - file_path = ali_oss_upload(file, file_name=validated_data['name']) - if file_path: - validated_data['file_url'] = file_path - else: - raise ValueError("上传失败") - elif file_engine == 'cos': - from dvadmin.utils.tencentcos import tencent_cos_upload - file_path = tencent_cos_upload(file, file_name=validated_data['name']) - if file_path: - validated_data['file_url'] = file_path - else: - raise ValueError("上传失败") + + # 上传到 RustFS + from dvadmin.utils.rustfs_storage import rustfs_upload_file + file_path = rustfs_upload_file(file, file_name=validated_data['name']) + if file_path: + validated_data['file_url'] = file_path else: - validated_data['url'] = file + raise ValueError("文件上传失败") + # 审计字段 try: request_user = self.request.user diff --git a/backend/dvadmin/utils/aliyunoss.py b/backend/dvadmin/utils/aliyunoss.py deleted file mode 100644 index b4e2894..0000000 --- a/backend/dvadmin/utils/aliyunoss.py +++ /dev/null @@ -1,62 +0,0 @@ -# -*- coding: utf-8 -*- - -import oss2 -from rest_framework.exceptions import ValidationError - -from application import dispatch - - -# 进度条 -# 当无法确定待上传的数据长度时,total_bytes的值为None。 -def percentage(consumed_bytes, total_bytes): - if total_bytes: - rate = int(100 * (float(consumed_bytes) / float(total_bytes))) - print('\r{0}% '.format(rate), end='') - - -def ali_oss_upload(file, file_name): - """ - 阿里云OSS上传 - """ - try: - file.seek(0) - file_read = file.read() - except Exception as e: - file_read = file - if not file: - raise ValidationError('请上传文件') - # 转存到oss - path_prefix = dispatch.get_system_config_values("file_storage.aliyun_path") - if not path_prefix.endswith('/'): - path_prefix = path_prefix + '/' - if path_prefix.startswith('/'): - path_prefix = path_prefix[1:] - base_fil_name = f'{path_prefix}{file_name}' - # 获取OSS配置 - # 获取的AccessKey - access_key_id = dispatch.get_system_config_values("file_storage.aliyun_access_key") - access_key_secret = dispatch.get_system_config_values("file_storage.aliyun_access_secret") - auth = oss2.Auth(access_key_id, access_key_secret) - # 这个是需要用特定的地址,不同地域的服务器地址不同,不要弄错了 - # 参考官网给的地址配置https://www.alibabacloud.com/help/zh/object-storage-service/latest/regions-and-endpoints#concept-zt4-cvy-5db - endpoint = dispatch.get_system_config_values("file_storage.aliyun_endpoint") - bucket_name = dispatch.get_system_config_values("file_storage.aliyun_bucket") - if bucket_name.endswith(endpoint): - bucket_name = bucket_name.replace(f'.{endpoint}', '') - # 你的项目名称,类似于不同的项目上传的图片前缀url不同 - bucket = oss2.Bucket(auth, endpoint, bucket_name) # 项目名称 - # 生成外网访问的文件路径 - aliyun_cdn_url = dispatch.get_system_config_values("file_storage.aliyun_cdn_url") - if aliyun_cdn_url: - if aliyun_cdn_url.endswith('/'): - aliyun_cdn_url = aliyun_cdn_url[1:] - file_path = f"{aliyun_cdn_url}/{base_fil_name}" - else: - file_path = f"https://{bucket_name}.{endpoint}/{base_fil_name}" - # 这个是阿里提供的SDK方法 - res = bucket.put_object(base_fil_name, file_read, progress_callback=percentage) - # 如果上传状态是200 代表成功 返回文件外网访问路径 - if res.status == 200: - return file_path - else: - return None diff --git a/backend/dvadmin/utils/rustfs_storage.py b/backend/dvadmin/utils/rustfs_storage.py new file mode 100644 index 0000000..00cce3d --- /dev/null +++ b/backend/dvadmin/utils/rustfs_storage.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +""" +RustFS (S3-compatible) 文件存储工具 +""" +import uuid +from typing import Optional + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError +from django.conf import settings + + +def get_rustfs_client(): + """ + 创建 RustFS S3 客户端 + """ + client = boto3.client( + 's3', + endpoint_url=f"{'https' if settings.RUSTFS_SECURE else 'http'}://{settings.RUSTFS_ENDPOINT}", + aws_access_key_id=settings.RUSTFS_ACCESS_KEY, + aws_secret_access_key=settings.RUSTFS_SECRET_KEY, + region_name=settings.RUSTFS_REGION or 'us-east-1', + config=Config(signature_version='s3v4'), + ) + return client + + +def rustfs_upload_file(file_obj, file_name: str) -> Optional[str]: + """ + 上传文件到 RustFS + + Args: + file_obj: 文件对象 (InMemoryUploadedFile 或 UploadedFile) + file_name: 原始文件名 + + Returns: + 文件访问URL,失败返回 None + """ + try: + # 生成唯一文件名 (uuid前缀避免冲突) + ext = file_name.split('.')[-1] if '.' in file_name else '' + unique_name = f"{uuid.uuid4().hex}.{ext}" if ext else uuid.uuid4().hex + + # 路径前缀处理 + path_prefix = getattr(settings, 'RUSTFS_PATH_PREFIX', '') + if path_prefix: + path_prefix = path_prefix.strip('/') + '/' + object_name = f"{path_prefix}{unique_name}" + + # 获取文件内容 + file_obj.seek(0) + file_content = file_obj.read() + + # 上传 + client = get_rustfs_client() + client.put_object( + Bucket=settings.RUSTFS_BUCKET, + Key=object_name, + Body=file_content, + ContentType=getattr(file_obj, 'content_type', 'application/octet-stream'), + ) + + # 生成访问URL + file_url = rustfs_get_file_url(object_name) + return file_url + + except ClientError as e: + print(f"RustFS upload error: {e}") + return None + except Exception as e: + print(f"RustFS upload unexpected error: {e}") + return None + + +def rustfs_delete_file(file_url: str) -> bool: + """ + 从 RustFS 删除文件 + + Args: + file_url: 文件完整URL或 object_key + + Returns: + 删除成功返回 True,否则 False + """ + try: + # 从URL中提取 object_key + object_key = file_url + if settings.RUSTFS_ENDPOINT in file_url: + # 提取 /bucket/key 部分 + parts = file_url.split(settings.RUSTFS_BUCKET + '/') + if len(parts) > 1: + object_key = parts[-1] + else: + return True # URL格式不对,当作已删除 + + client = get_rustfs_client() + client.delete_object(Bucket=settings.RUSTFS_BUCKET, Key=object_key) + return True + except ClientError as e: + print(f"RustFS delete error: {e}") + return False + + +def rustfs_get_file_url(object_key: str) -> str: + """ + 获取文件访问URL + + Args: + object_key: 对象存储的key + + Returns: + 完整的文件访问URL + """ + scheme = 'https' if settings.RUSTFS_SECURE else 'http' + endpoint = settings.RUSTFS_ENDPOINT + bucket = settings.RUSTFS_BUCKET + return f"{scheme}://{endpoint}/{bucket}/{object_key}" diff --git a/backend/dvadmin/utils/tencentcos.py b/backend/dvadmin/utils/tencentcos.py deleted file mode 100644 index a515124..0000000 --- a/backend/dvadmin/utils/tencentcos.py +++ /dev/null @@ -1,56 +0,0 @@ -# -*- coding: utf-8 -*- -from rest_framework.exceptions import ValidationError - -from application import dispatch -from qcloud_cos import CosConfig -from qcloud_cos import CosS3Client - - -# 进度条 -# 当无法确定待上传的数据长度时,total_bytes的值为None。 -def percentage(consumed_bytes, total_bytes): - if total_bytes: - rate = int(100 * (float(consumed_bytes) / float(total_bytes))) - print('\r{0}% '.format(rate), end='') - -def tencent_cos_upload(file, file_name): - try: - file.seek(0) - file_read = file.read() - except Exception as e: - file_read = file - if not file: - raise ValidationError('请上传文件') - # 生成文件名 - path_prefix = dispatch.get_system_config_values("file_storage.tencent_path") - if not path_prefix.endswith('/'): - path_prefix = path_prefix + '/' - if path_prefix.startswith('/'): - path_prefix = path_prefix[1:] - base_fil_name = f'{path_prefix}{file_name}' - # 获取cos配置 - # 1. 设置用户属性, 包括 secret_id, secret_key, region等。Appid 已在 CosConfig 中移除,请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 - secret_id = dispatch.get_system_config_values("file_storage.tencent_secret_id") # 用户的 SecretId,建议使用子账号密钥,授权遵循最小权限指引,降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 - secret_key = dispatch.get_system_config_values("file_storage.tencent_secret_key") # 用户的 SecretKey,建议使用子账号密钥,授权遵循最小权限指引,降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 - region = dispatch.get_system_config_values("file_storage.tencent_region") # 替换为用户的 region,已创建桶归属的 region 可以在控制台查看,https://console.cloud.tencent.com/cos5/bucket # COS 支持的所有 region 列表参见https://cloud.tencent.com/document/product/436/6224 - bucket = dispatch.get_system_config_values("file_storage.tencent_bucket") # 要访问的桶名称 - config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key) - client = CosS3Client(config) - # 访问地址 - base_file_url = f'https://{bucket}.cos.{region}.myqcloud.com' - # 生成外网访问的文件路径 - if base_file_url.endswith('/'): - file_path = base_file_url + base_fil_name - else: - file_path = f'{base_file_url}/{base_fil_name}' - # 这个是阿里提供的SDK方法 bucket是调用的4.1中配置的变量名 - try: - response = client.put_object( - Bucket=bucket, - Body=file_read, - Key=base_fil_name, - EnableMD5=False - ) - return file_path - except: - return None diff --git a/backend/requirements.txt b/backend/requirements.txt index 6c4ae3a..cd5b9a0 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -29,8 +29,9 @@ gevent==24.2.1 Pillow==10.4.0 pyinstaller==6.9.0 dvadmin3-celery==3.1.6 -oss2==2.19.1 -cos-python-sdk-v5==1.9.37 +boto3>=1.34.0 +pytest-django>=4.5.0 +factory_boy>=3.3.0 # Git hooks pre-commit>=3.0.0 diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 017389e..a5cede5 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -18,24 +18,16 @@ def authenticate(api_client, admin_user): @pytest.fixture def admin_user(db): """创建管理员用户(每个测试独立事务,测试后自动 rollback)""" - from dvadmin.system.models import Users + from tests.factories.system_factory import UserFactory - user = Users.objects.create_user( - username="test_admin", - password="testpass123", - is_superuser=True, - ) + user = UserFactory(admin=True) return user @pytest.fixture def normal_user(db): """创建普通用户""" - from dvadmin.system.models import Users + from tests.factories.system_factory import UserFactory - user = Users.objects.create_user( - username="test_normal_user", - password="testpass123", - is_superuser=False, - ) + user = UserFactory() return user diff --git a/backend/tests/factories/file_factory.py b/backend/tests/factories/file_factory.py new file mode 100644 index 0000000..9ca9e55 --- /dev/null +++ b/backend/tests/factories/file_factory.py @@ -0,0 +1,13 @@ +import factory +from dvadmin.system.models import FileList + + +class FileListFactory(factory.django.DjangoModelFactory): + class Meta: + model = FileList + + name = factory.Sequence(lambda n: f"test_file_{n}.txt") + engine = "rustfs" + mime_type = "text/plain" + size = "100" + md5sum = factory.Faker("md5") diff --git a/backend/tests/test_file_upload.py b/backend/tests/test_file_upload.py new file mode 100644 index 0000000..8a334d5 --- /dev/null +++ b/backend/tests/test_file_upload.py @@ -0,0 +1,83 @@ +""" +RustFS 文件上传集成测试 + +依赖: 真实的 RustFS 服务运行在 RUSTFS_ENDPOINT (192.168.80.90:9000) +测试前确保: + 1. RustFS 服务已启动 + 2. bucket "pis-media" 已创建 + 3. RUSTFS_ACCESS_KEY / RUSTFS_SECRET_KEY 正确配置 +""" +import io +import pytest +from django.conf import settings +from rest_framework import status + + +CODE_SUCCESS = 2000 +CODE_ERROR = 4000 + + +@pytest.mark.django_db +class TestFileUpload: + """文件上传 API 测试""" + + def test_upload_file_success(self, authenticate): + """POST 上传文件返回成功 + RustFS URL""" + file_content = b"Hello, RustFS!" + file_obj = io.BytesIO(file_content) + file_obj.name = "test.txt" + + response = authenticate.post( + "/api/system/file/", + data={"file": file_obj}, + format="multipart", + ) + + assert response.data["code"] == CODE_SUCCESS, f"上传失败: {response.data}" + assert "file_url" in response.data["data"] + file_url = response.data["data"]["file_url"] + # 验证 URL 指向 RustFS + assert settings.RUSTFS_ENDPOINT in file_url, f"文件 URL 非 RustFS 地址: {file_url}" + assert file_url.startswith("http://") or file_url.startswith("https://") + + def test_upload_file_unauthenticated(self, api_client): + """未认证请求上传文件返回错误码""" + file_content = b"Unauthorized file" + file_obj = io.BytesIO(file_content) + file_obj.name = "unauth.txt" + + response = api_client.post( + "/api/system/file/", + data={"file": file_obj}, + format="multipart", + ) + # 无认证时返回 4000 或非 2000 + assert response.data["code"] == CODE_ERROR + + def test_upload_image_success(self, authenticate): + """POST 上传图片返回成功 + image file_type""" + # GIF 格式最小文件 (1x1 透明像素) + gif_bytes = ( + b"\x47\x49\x46\x38\x39\x61\x01\x00\x01\x00\x80\x00\x00" + b"\xff\xff\xff\x00\x00\x00\x21\xf9\x04\x01\x00\x00\x00" + b"\x00\x2c\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02\x44" + b"\x01\x00\x3b" + ) + file_obj = io.BytesIO(gif_bytes) + file_obj.name = "test.gif" + + response = authenticate.post( + "/api/system/file/", + data={"file": file_obj}, + format="multipart", + ) + + assert response.data["code"] == CODE_SUCCESS, f"图片上传失败: {response.data}" + file_url = response.data["data"]["file_url"] + assert settings.RUSTFS_ENDPOINT in file_url + + def test_upload_without_file(self, authenticate): + """POST 不带文件返回错误码""" + response = authenticate.post("/api/system/file/", data={}, format="multipart") + # 不带文件应该失败 + assert response.data["code"] == CODE_ERROR