Add multi level caching and optimize caching (#1054)

* Add multi level caching and optimize caching

* optimize current caching

* improved decorators

* improved local

* add pub/sub

* fix serialize

* Improve pub and sub

* Add comment

* Fix lru_cache maxsize

* Fix lint

* Fix config warmup

backend/plugin/config/service/config_service.py

@@ -3,8 +3,10 @@ from typing import Any
 
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from backend.common.cache.decorator import cache_invalidate, cached
 from backend.common.exception import errors
 from backend.common.pagination import paging_data
+from backend.core.conf import settings
 from backend.plugin.config.crud.crud_config import config_dao
 from backend.plugin.config.model import Config
 from backend.plugin.config.schema.config import (
@@ -26,13 +28,16 @@ class ConfigService:
         :param pk: 参数配置 ID
         :return:
         """
-
         config = await config_dao.get(db, pk)
         if not config:
             raise errors.NotFoundError(msg='参数配置不存在')
         return config
 
     @staticmethod
+    @cached(
+        settings.CACHE_CONFIG_REDIS_PREFIX,
+        key_builder=lambda *, db, type: f'type:{type}',
+    )
     async def get_all(*, db: AsyncSession, type: str | None) -> Sequence[Config | None]:
         """
         获取所有参数配置
@@ -41,7 +46,6 @@ class ConfigService:
         :param type: 参数配置类型
         :return:
         """
-
         return await config_dao.get_all(db, type)
 
     @staticmethod
@@ -58,6 +62,7 @@ class ConfigService:
         return await paging_data(db, config_select)
 
     @staticmethod
+    @cache_invalidate(settings.CACHE_CONFIG_REDIS_PREFIX)
     async def create(*, db: AsyncSession, obj: CreateConfigParam) -> None:
         """
         创建参数配置
@@ -66,13 +71,13 @@ class ConfigService:
         :param obj: 参数配置创建参数
         :return:
         """
-
         config = await config_dao.get_by_key(db, obj.key)
         if config:
             raise errors.ConflictError(msg=f'参数配置 {obj.key} 已存在')
         await config_dao.create(db, obj)
 
     @staticmethod
+    @cache_invalidate(settings.CACHE_CONFIG_REDIS_PREFIX)
     async def update(*, db: AsyncSession, pk: int, obj: UpdateConfigParam) -> int:
         """
         更新参数配置
@@ -82,7 +87,6 @@ class ConfigService:
         :param obj: 参数配置更新参数
         :return:
         """
-
         config = await config_dao.get(db, pk)
         if not config:
             raise errors.NotFoundError(msg='参数配置不存在')
@@ -94,6 +98,7 @@ class ConfigService:
         return count
 
     @staticmethod
+    @cache_invalidate(settings.CACHE_CONFIG_REDIS_PREFIX)
     async def bulk_update(*, db: AsyncSession, objs: list[UpdateConfigsParam]) -> int:
         """
         批量更新参数配置
@@ -102,7 +107,6 @@ class ConfigService:
         :param objs: 参数配置批量更新参数
         :return:
         """
-
         for _batch in range(0, len(objs), 1000):
             for obj in objs:
                 config = await config_dao.get(db, obj.id)
@@ -116,6 +120,7 @@ class ConfigService:
         return count
 
     @staticmethod
+    @cache_invalidate(settings.CACHE_CONFIG_REDIS_PREFIX)
     async def delete(*, db: AsyncSession, pks: list[int]) -> int:
         """
         批量删除参数配置
@@ -124,7 +129,6 @@ class ConfigService:
         :param pks: 参数配置 ID 列表
         :return:
         """
-
         count = await config_dao.delete(db, pks)
         return count
 

backend/plugin/core.py

@@ -22,8 +22,8 @@ from backend.utils.async_helper import run_await
 from backend.utils.dynamic_import import get_model_objects, import_module_cached
 
 
-@lru_cache
-def get_plugins() -> list[str]:
+@lru_cache(maxsize=128)
+def get_plugins() -> tuple[str, ...]:
     """获取插件列表"""
     plugin_packages = []
 
@@ -37,7 +37,7 @@ def get_plugins() -> list[str]:
         if os.path.isdir(item_path) and '__init__.py' in os.listdir(item_path):
             plugin_packages.append(item)
 
-    return plugin_packages
+    return tuple(plugin_packages)
 
 
 def get_plugin_models() -> list[object]:
@@ -104,7 +104,7 @@ def parse_plugin_config() -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
 
     plugins = get_plugins()
 
-    # 使用独立单例，避免与主线程冲突
+    # 使用独立连接
     current_redis_client = RedisCli()
     run_await(current_redis_client.init)()
 

backend/plugin/dict/service/dict_data_service.py

@@ -3,8 +3,10 @@ from typing import Any
 
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from backend.common.cache.decorator import cache_invalidate, cached
 from backend.common.exception import errors
 from backend.common.pagination import paging_data
+from backend.core.conf import settings
 from backend.plugin.dict.crud.crud_dict_data import dict_data_dao
 from backend.plugin.dict.crud.crud_dict_type import dict_type_dao
 from backend.plugin.dict.model import DictData
@@ -23,13 +25,16 @@ class DictDataService:
         :param pk: 字典数据 ID
         :return:
         """
-
         dict_data = await dict_data_dao.get(db, pk)
         if not dict_data:
             raise errors.NotFoundError(msg='字典数据不存在')
         return dict_data
 
     @staticmethod
+    @cached(
+        settings.CACHE_DICT_REDIS_PREFIX,
+        key_builder=lambda *, db, code: f'type:{code}',
+    )
     async def get_by_type_code(*, db: AsyncSession, code: str) -> Sequence[DictData]:
         """
         获取字典数据详情
@@ -38,13 +43,16 @@ class DictDataService:
         :param code: 字典类型编码
         :return:
         """
-
         dict_datas = await dict_data_dao.get_by_type_code(db, code)
         if not dict_datas:
             raise errors.NotFoundError(msg='字典数据不存在')
         return dict_datas
 
     @staticmethod
+    @cached(
+        settings.CACHE_DICT_REDIS_PREFIX,
+        key_builder=lambda *, db: 'all',
+    )
     async def get_all(*, db: AsyncSession) -> Sequence[DictData]:
         """
         获取所有字典数据
@@ -86,6 +94,7 @@ class DictDataService:
         return await paging_data(db, dict_data_select)
 
     @staticmethod
+    @cache_invalidate(settings.CACHE_DICT_REDIS_PREFIX)
     async def create(*, db: AsyncSession, obj: CreateDictDataParam) -> None:
         """
         创建字典数据
@@ -103,6 +112,7 @@ class DictDataService:
         await dict_data_dao.create(db, obj, dict_type.code)
 
     @staticmethod
+    @cache_invalidate(settings.CACHE_DICT_REDIS_PREFIX)
     async def update(*, db: AsyncSession, pk: int, obj: UpdateDictDataParam) -> int:
         """
         更新字典数据
@@ -112,7 +122,6 @@ class DictDataService:
         :param obj: 字典数据更新参数
         :return:
         """
-
         dict_data = await dict_data_dao.get(db, pk)
         if not dict_data:
             raise errors.NotFoundError(msg='字典数据不存在')
@@ -127,6 +136,7 @@ class DictDataService:
         return count
 
     @staticmethod
+    @cache_invalidate(settings.CACHE_DICT_REDIS_PREFIX)
     async def delete(*, db: AsyncSession, obj: DeleteDictDataParam) -> int:
         """
         批量删除字典数据
@@ -135,7 +145,6 @@ class DictDataService:
         :param obj: 字典数据 ID 列表
         :return:
         """
-
         count = await dict_data_dao.delete(db, obj.pks)
         return count
 

backend/plugin/installer.py

@@ -104,7 +104,7 @@ async def install_zip_plugin(file: UploadFile | str) -> str:
 
         await _append_env_example(full_plugin_path)
         await install_requirements_async(plugin_dir_name)
-        await redis_client.set(f'{settings.PLUGIN_REDIS_PREFIX}:changed', 'ture')
+        await redis_client.set(f'{settings.PLUGIN_REDIS_PREFIX}:changed', 'true')
 
     return plugin_name
 
@@ -133,6 +133,6 @@ async def install_git_plugin(repo_url: str) -> str:
 
         await _append_env_example(path)
         await install_requirements_async(repo_name)
-        await redis_client.set(f'{settings.PLUGIN_REDIS_PREFIX}:changed', 'ture')
+        await redis_client.set(f'{settings.PLUGIN_REDIS_PREFIX}:changed', 'true')
 
     return repo_name

backend/plugin/requirements.py

@@ -9,20 +9,10 @@ from starlette.concurrency import run_in_threadpool
 
 from backend.core.conf import settings
 from backend.core.path_conf import PLUGIN_DIR
+from backend.plugin.core import get_plugins
 from backend.plugin.errors import PluginInstallError
 
 
-def get_plugins() -> list[str]:
-    """
-    获取插件列表
-
-    注意：此函数从 backend.plugin.core 导入以避免循环依赖
-    """
-    from backend.plugin.core import get_plugins as _get_plugins
-
-    return _get_plugins()
-
-
 def _is_in_virtualenv() -> bool:
     """检测当前是否在虚拟环境中运行"""
     return hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix)