RuoYi-Vue3-FastAPI/ruoyi-fastapi-backend/plugins/core/runtime/application.py

import asyncio
import json
import time
from collections.abc import Awaitable, Callable, Mapping
from functools import cache

from fastapi import FastAPI

from common.constant import LockConstant
from plugins.core.runtime.diagnostics_store import PluginDiagnosticsReporter, diagnostics_namespace
from plugins.core.runtime.explicit import ExplicitPluginRuntime
from plugins.core.runtime.startup import PluginRuntimeStartupManager
from plugins.core.runtime.startup_coordination import PluginStartupGenerationResolver
from utils.log_util import logger

from .service.lifecycle_lock import NoopPluginLifecycleLock, PluginLifecycleLock


class PluginApplicationRuntime:
    """
    应用入口侧插件运行时适配器。

    server.py 只依赖该适配器提供的高层扩展点，插件实体导入、二次建表、启动资源安装、
    多 worker ready barrier 和生命周期钩子等细节留在插件运行时内部。
    """

    def __init__(
        self,
        startup_manager: PluginRuntimeStartupManager | None = None,
        *,
        ready_key: str = LockConstant.PLUGIN_STARTUP_READY_KEY,
        ready_expire_seconds: int = LockConstant.PLUGIN_STARTUP_READY_EXPIRE_SECONDS,
        failed_expire_seconds: int = LockConstant.PLUGIN_STARTUP_FAILED_EXPIRE_SECONDS,
        ready_wait_timeout_seconds: int = LockConstant.PLUGIN_STARTUP_READY_WAIT_TIMEOUT_SECONDS,
        ready_wait_interval_seconds: int = LockConstant.PLUGIN_STARTUP_READY_WAIT_INTERVAL_SECONDS,
        lifecycle_lock: PluginLifecycleLock | None = None,
        startup_generation: str | None = None,
        release_worker: object | None = None,
    ) -> None:
        """
        初始化插件应用运行时适配器。

        :param startup_manager: 插件启动协调器
        :param ready_key: 插件启动 ready 标记 key
        :param ready_expire_seconds: ready 标记过期时间
        :param failed_expire_seconds: 失败标记过期时间
        :param ready_wait_timeout_seconds: 等待 ready 超时时间
        :param ready_wait_interval_seconds: 等待 ready 轮询间隔
        :param lifecycle_lock: 插件生命周期全局锁
        :param startup_generation: 测试或部署显式注入的启动代际
        :param release_worker: 维护当前发布快照和 worker 心跳的可选协调器
        :return: None
        """
        self.startup_manager = startup_manager or PluginRuntimeStartupManager()
        self.ready_key = ready_key
        self.ready_expire_seconds = ready_expire_seconds
        self.failed_expire_seconds = failed_expire_seconds
        self.ready_wait_timeout_seconds = ready_wait_timeout_seconds
        self.ready_wait_interval_seconds = ready_wait_interval_seconds
        self.lifecycle_lock = lifecycle_lock or NoopPluginLifecycleLock()
        self.startup_generation = startup_generation
        self.release_worker = release_worker

    def bind_app(self, app: FastAPI) -> None:
        """
        绑定插件运行时到 FastAPI app。

        :param app: FastAPI对象
        :return: None
        """
        app.state.plugin_application_runtime = self
        self.startup_manager.bind_app(app)

    def prepare_metadata(self, app: FastAPI) -> None:
        """
        准备插件平台自身元数据。

        :param app: FastAPI对象
        :return: None
        """
        self._ensure_bound(app)
        self.startup_manager.import_builtin_entities()
        if self.release_worker is not None:
            from plugins.core.management.entity.do import release_models  # noqa: F401, PLC0415

    async def startup(
        self,
        app: FastAPI,
        *,
        create_tables: Callable[[], Awaitable[None]],
    ) -> None:
        """
        启动插件运行时。

        同一发布代际只有一个 worker 能在全局生命周期锁内执行写入；其他 worker 等待
        当前代际 ready 后仅执行进程本地激活。插件启动不再依赖长期持有的调度器主节点锁。

        :param app: FastAPI对象
        :param create_tables: 数据库建表回调
        :return: None
        """
        try:
            if self.release_worker is not None:
                await self.release_worker.start(
                    app, self.startup_manager.builder, self.lifecycle_lock, timeout=self.ready_wait_timeout_seconds
                )
                self.startup_generation = self.release_worker.generation(self.resolve_startup_generation())
                app.state.plugin_release_worker = self.release_worker
            await self._startup_coordinated(app, create_tables=create_tables)
            if self.release_worker is not None:
                await self.release_worker.ready()
            runtime = getattr(app.state, 'plugin_explicit_runtime', None)
            if isinstance(runtime, ExplicitPluginRuntime):
                if self.release_worker is not None:
                    # 即使全部显式插件准备失败，空运行时也属于已有发布 worker。
                    runtime.metrics.worker_id = self.release_worker.worker_id
                runtime.metrics_reporter.start(getattr(app.state, 'redis', None), f'{self.ready_key}:metrics')
                reporter = PluginDiagnosticsReporter(runtime.diagnostic_snapshot, runtime.metrics.worker_id)
                app.state.plugin_diagnostics_reporter = reporter
                reporter.start(getattr(app.state, 'redis', None), diagnostics_namespace(self.ready_key))
        except BaseException:
            # 激活后的 ready 发布或锁释放也可能失败，此时 server 尚未标记启动完成。
            # 显式资源需在这里释放，不能依赖正常 shutdown 分支。
            runtime = getattr(app.state, 'plugin_explicit_runtime', None)
            try:
                if isinstance(runtime, ExplicitPluginRuntime):
                    await runtime.shutdown()
            finally:
                await self._stop_diagnostics(app)
                if self.release_worker is not None:
                    try:
                        await self.release_worker.stop(failed=True)
                    except Exception:
                        logger.exception('插件启动失败后的 worker 状态上报失败；旧心跳将按 TTL 失效')
            raise

    async def _startup_coordinated(
        self,
        app: FastAPI,
        *,
        create_tables: Callable[[], Awaitable[None]],
    ) -> None:
        """
        执行 writer/reader 协调，包括激活后的 ready 发布及锁释放。

        :param app: 宿主 FastAPI 应用
        :param create_tables: 在允许写入的 worker 中创建数据库表的异步回调
        :return: None
        """
        self._ensure_bound(app)
        if self.release_worker is not None:
            await self.release_worker.assert_snapshot_current()
        generation = self.resolve_startup_generation()
        app.state.plugin_startup_generation = generation
        app.state.plugin_startup_write_enabled = False

        ready_state = await self.get_startup_state(app, generation)
        logger.bind(
            startup_generation=generation,
            plugin_startup_role='reader',
            ready_status=ready_state.get('status', 'missing'),
        ).debug('🔎 开始检查插件启动代际 ready 状态')
        activated, stale_ready_reported = await self._process_startup_state(
            app,
            generation,
            ready_state,
            stale_ready_reported=False,
            ready_message='复用当前代际插件 ready，开始本地激活',
            failed_message='检测到当前代际插件启动 failed marker',
        )
        if activated:
            return

        deadline = time.monotonic() + self.ready_wait_timeout_seconds
        wait_reported = False
        while time.monotonic() < deadline:
            async with self.lifecycle_lock.lock('__runtime__', f'startup:{generation}') as lock_result:
                if lock_result.acquired:
                    ready_state = await self.get_startup_state(app, generation)
                    activated, stale_ready_reported = await self._process_startup_state(
                        app,
                        generation,
                        ready_state,
                        stale_ready_reported=stale_ready_reported,
                        ready_message='锁内复用当前代际插件 ready，开始本地激活',
                        failed_message='锁内检测到当前代际插件启动 failed marker',
                    )
                    if activated:
                        return
                    logger.bind(
                        startup_generation=generation,
                        plugin_startup_role='writer',
                        ready_status=ready_state.get('status', 'missing'),
                    ).info('🎯 当前 worker 成为插件启动 writer')
                    await self._run_startup_writer(app, generation, create_tables)
                    return

            ready_state = await self.get_startup_state(app, generation)
            activated, stale_ready_reported = await self._process_startup_state(
                app,
                generation,
                ready_state,
                stale_ready_reported=stale_ready_reported,
                ready_message='等待结束，复用当前代际插件 ready',
                failed_message='等待期间检测到当前代际插件启动 failed marker',
            )
            if activated:
                return
            if not wait_reported:
                logger.bind(
                    startup_generation=generation,
                    plugin_startup_role='reader',
                    ready_status=ready_state.get('status', 'missing'),
                ).debug('⏳ 等待插件启动 writer 发布当前代际 ready')
                wait_reported = True
            await asyncio.sleep(self.ready_wait_interval_seconds)

        logger.bind(
            startup_generation=generation,
            plugin_startup_role='reader',
            ready_status=ready_state.get('status', 'missing'),
        ).error('❌ 等待插件启动 ready 超时')
        raise TimeoutError(f'等待插件启动代际 {generation} ready 超时')

    async def _process_startup_state(
        self,
        app: FastAPI,
        generation: str,
        ready_state: Mapping[str, str],
        *,
        stale_ready_reported: bool,
        ready_message: str,
        failed_message: str,
    ) -> tuple[bool, bool]:
        """
        处理一次插件启动状态读取结果。

        :param app: FastAPI对象
        :param generation: 插件启动代际
        :param ready_state: 当前ready状态
        :param stale_ready_reported: 是否已记录过期ready告警
        :param ready_message: 复用ready时的日志
        :param failed_message: 发现failed marker时的日志
        :return: 是否已完成reader激活、是否已记录过期ready告警
        """
        ready_status = ready_state.get('status', 'missing')
        reader_logger = logger.bind(
            startup_generation=generation,
            plugin_startup_role='reader',
            ready_status=ready_status,
        )
        if ready_status == 'success':
            if await self.requires_startup_write():
                if not stale_ready_reported:
                    logger.bind(
                        startup_generation=generation,
                        plugin_startup_role='reader',
                        ready_status=ready_status,
                        stale_ready_ignored=True,
                        startup_write_required=True,
                        startup_write_reason='missing_default_plugin_state',
                    ).warning('⚠️ 数据库默认插件状态缺失，忽略当前代际旧 ready 标记')
                return False, True
            logger.bind(
                startup_generation=generation,
                plugin_startup_role='reader',
                ready_status=ready_status,
                plugin_install_lifecycle='skipped',
                plugin_resource_sync='skipped',
                plugin_entity_table_sync='skipped',
            ).info(
                '⏭️ 复用插件 ready 状态，跳过启动期全局写入：'
                'plugin_install_lifecycle=skipped，'
                'plugin_resource_sync=skipped，'
                'plugin_entity_table_sync=skipped'
            )
            reader_logger.debug(f'🔄 {ready_message}')
            await self._activate_startup_reader(app, generation)
            return True, stale_ready_reported
        if ready_status == 'failed':
            reader_logger.error(f'❌ {failed_message}')
            raise RuntimeError(self._build_startup_failure_message(ready_state))
        return False, stale_ready_reported

    async def requires_startup_write(self) -> bool:
        """
        校验 ready 标记对应的数据库状态是否仍然完整。

        测试替身或旧适配器未实现该能力时保持原有行为；生产启动管理器会检查
        默认启用插件是否仍具备安装状态。

        :return: 是否必须重新执行启动期写入
        """
        checker = getattr(type(self.startup_manager), 'requires_startup_write', None)
        if checker is None:
            return False
        return bool(await self.startup_manager.requires_startup_write())

    async def _run_startup_writer(
        self,
        app: FastAPI,
        generation: str,
        create_tables: Callable[[], Awaitable[None]],
    ) -> None:
        """
        执行当前代际唯一的插件启动写入。

        :param app: FastAPI对象
        :param generation: 插件启动代际
        :param create_tables: 数据库建表回调
        :return: None
        """
        app.state.plugin_startup_write_enabled = True
        writer_logger = logger.bind(
            startup_generation=generation,
            plugin_startup_role='writer',
            ready_status='initializing',
        )
        writer_logger.info('🔄 开始同步插件全局启动资源')
        await self.clear_startup_ready(app, generation)
        try:
            await self.startup_manager.prepare_enabled_plugins(app, startup_write_enabled=True)
            await create_tables()
            await self.startup_manager.activate_enabled_plugins(app, startup_write_enabled=True)
        except Exception as exc:
            await self.mark_startup_failed(app, generation, exc)
            logger.bind(
                startup_generation=generation,
                plugin_startup_role='writer',
                ready_status='failed',
            ).exception('❌ 插件全局启动资源同步失败，已写入 failed marker')
            raise
        await self.mark_startup_ready(app, generation)
        logger.bind(
            startup_generation=generation,
            plugin_startup_role='writer',
            ready_status='success',
        ).info('✅ 插件全局启动资源同步完成')
        self._log_startup_completed(app, generation, role='writer')

    async def _activate_startup_reader(self, app: FastAPI, generation: str) -> None:
        """
        在当前 worker 执行无全局写入的本地插件激活。

        :param app: FastAPI对象
        :param generation: 插件启动代际
        :return: None
        """
        reader_logger = logger.bind(
            startup_generation=generation,
            plugin_startup_role='reader',
            ready_status='success',
        )
        reader_logger.debug('🔄 开始执行插件本地实体、Hook和路由激活')
        await self.startup_manager.prepare_enabled_plugins(app, startup_write_enabled=False)
        await self.startup_manager.activate_enabled_plugins(app, startup_write_enabled=False)
        reader_logger.debug('✅ 插件本地实体、Hook和路由激活完成')
        self._log_startup_completed(app, generation, role='reader')

    @staticmethod
    def _log_startup_completed(app: FastAPI, generation: str, *, role: str) -> None:
        """
        为每个成功启动的worker输出一条稳定可见的插件运行时摘要。

        writer和reader内部过程仍按原有级别记录；该INFO摘要用于避免命中ready后
        只有DEBUG日志而表现为插件日志随机缺失。

        :param app: FastAPI对象
        :param generation: 插件启动代际
        :param role: 当前worker的插件启动角色
        :return: None
        """
        plugin_registry = getattr(app.state, 'plugin_registry', None)
        enabled_plugin_ids = (
            sorted(str(plugin.plugin_id) for plugin in plugin_registry.list_enabled_plugins())
            if plugin_registry is not None
            else []
        )
        enabled_plugins = ','.join(enabled_plugin_ids) or 'none'
        logger.bind(
            startup_generation=generation,
            plugin_startup_role=role,
            ready_status='success',
            enabled_plugin_ids=enabled_plugin_ids,
            enabled_plugin_count=len(enabled_plugin_ids),
        ).info(f'✅ 插件运行时启动完成：role={role}，generation={generation[:8]}，enabled={enabled_plugins}')

    async def shutdown(self, app: FastAPI) -> None:
        """
        关闭插件运行时。

        :param app: FastAPI对象
        :return: None
        """
        self._ensure_bound(app)
        startup_write_enabled = bool(getattr(app.state, 'plugin_startup_write_enabled', False))
        try:
            await self.startup_manager.shutdown(app, startup_write_enabled=startup_write_enabled)
        finally:
            await self._stop_diagnostics(app)
            if self.release_worker is not None:
                await self.release_worker.stop()

    @staticmethod
    async def _stop_diagnostics(app: FastAPI) -> None:
        """
        关闭诊断采样，观测清理失败不阻止发布 worker 上报最终状态。

        :param app: 宿主 FastAPI 应用
        :return: None
        """
        reporter = getattr(app.state, 'plugin_diagnostics_reporter', None)
        if isinstance(reporter, PluginDiagnosticsReporter):
            try:
                await reporter.stop()
            except Exception:
                logger.warning('插件运行诊断上报关闭失败，远程快照将按 TTL 过期')

    async def clear_startup_ready(self, app: FastAPI, generation: str | None = None) -> None:
        """
        清除指定代际的插件启动 ready 标记。

        :param app: FastAPI对象
        :param generation: 插件启动代际
        :return: None
        """
        resolved_generation = generation or self.resolve_startup_generation()
        await app.state.redis.delete(self.build_ready_key(resolved_generation))

    async def mark_startup_ready(self, app: FastAPI, generation: str | None = None) -> None:
        """
        标记指定代际已完成插件启动写入。

        :param app: FastAPI对象
        :param generation: 插件启动代际
        :return: None
        """
        resolved_generation = generation or self.resolve_startup_generation()
        await app.state.redis.set(
            self.build_ready_key(resolved_generation),
            json.dumps({'generation': resolved_generation, 'status': 'success'}, ensure_ascii=False),
            ex=self.ready_expire_seconds,
        )
        writer_logger = logger.bind(
            startup_generation=resolved_generation,
            plugin_startup_role='writer',
            ready_status='success',
        )
        failed_plugin_ids = getattr(app.state, 'plugin_dependency_failed_plugin_ids', set())
        if failed_plugin_ids:
            writer_logger.warning(
                f'⚠️ 插件启动协调已完成，依赖检查失败插件已隔离：{"、".join(sorted(failed_plugin_ids))}'
            )
            return
        writer_logger.info('✅ 插件启动资源已就绪')

    async def mark_startup_failed(self, app: FastAPI, generation: str, error: Exception) -> None:
        """
        标记指定插件启动代际初始化失败。

        :param app: FastAPI对象
        :param generation: 插件启动代际
        :param error: 启动异常
        :return: None
        """
        await app.state.redis.set(
            self.build_ready_key(generation),
            json.dumps(
                {
                    'generation': generation,
                    'status': 'failed',
                    'error': str(error)[:1000],
                },
                ensure_ascii=False,
            ),
            ex=self.failed_expire_seconds,
        )

    async def get_startup_state(self, app: FastAPI, generation: str) -> dict[str, str]:
        """
        读取指定代际的插件启动状态。

        :param app: FastAPI对象
        :param generation: 插件启动代际
        :return: 启动状态
        """
        raw_state = await app.state.redis.get(self.build_ready_key(generation))
        if not raw_state:
            return {}
        try:
            state = json.loads(raw_state)
        except (TypeError, json.JSONDecodeError):
            return {}
        if not isinstance(state, dict) or state.get('generation') != generation:
            return {}
        return {str(key): str(value) for key, value in state.items()}

    def resolve_startup_generation(self) -> str:
        """
        解析当前插件启动代际。

        :return: 插件启动代际
        """
        if self.startup_generation:
            return self.startup_generation
        generation_resolver = PluginStartupGenerationResolver(self.startup_manager.builder.backend_root)
        self.startup_generation = generation_resolver.resolve()
        return self.startup_generation

    def build_ready_key(self, generation: str) -> str:
        """
        构建代际隔离的 ready key。

        :param generation: 插件启动代际
        :return: Redis key
        """
        return f'{self.ready_key}:{generation}'

    @staticmethod
    def _build_startup_failure_message(state: Mapping[str, str]) -> str:
        """
        构建启动失败错误消息。

        :param state: 启动状态
        :return: 错误消息
        """
        error_message = state.get('error') or '未知错误'
        return f'插件启动代际 {state.get("generation", "")} 初始化失败：{error_message}'

    def _ensure_bound(self, app: FastAPI) -> None:
        """
        确保插件运行时已绑定到 app。

        :param app: FastAPI对象
        :return: None
        """
        if getattr(app.state, 'plugin_application_runtime', None) is not self:
            self.bind_app(app)


@cache
def get_plugin_application_runtime() -> PluginApplicationRuntime:
    """
    获取应用插件运行时适配器。

    :return: 应用插件运行时适配器
    """
    from plugins.core.deployment.config import PluginDeploymentConfig  # noqa: PLC0415
    from plugins.core.management.service.startup_gateway import (  # noqa: PLC0415
        PluginManagementRouteStateGateway,
        PluginManagementStartupGateway,
    )
    from plugins.core.runtime.service.lifecycle_lock import RedisPluginLifecycleLock  # noqa: PLC0415

    startup_manager = PluginRuntimeStartupManager(
        management_gateway=PluginManagementStartupGateway(),
        route_state_gateway=PluginManagementRouteStateGateway(),
    )
    deployment_config = PluginDeploymentConfig.from_settings(startup_manager.builder.backend_root)
    release_worker = None
    if deployment_config.enabled:
        from plugins.core.deployment.worker import PluginReleaseWorker  # noqa: PLC0415

        release_worker = PluginReleaseWorker(deployment_config)
    return PluginApplicationRuntime(
        startup_manager=startup_manager,
        lifecycle_lock=RedisPluginLifecycleLock(),
        release_worker=release_worker,
    )