import asyncio import json import time from collections.abc import Awaitable, Callable, Mapping from functools import cache from fastapi import FastAPI from common.constant import LockConstant from plugins.core.runtime.diagnostics_store import PluginDiagnosticsReporter, diagnostics_namespace from plugins.core.runtime.explicit import ExplicitPluginRuntime from plugins.core.runtime.startup import PluginRuntimeStartupManager from plugins.core.runtime.startup_coordination import PluginStartupGenerationResolver from utils.log_util import logger from .service.lifecycle_lock import NoopPluginLifecycleLock, PluginLifecycleLock class PluginApplicationRuntime: """ 应用入口侧插件运行时适配器。 server.py 只依赖该适配器提供的高层扩展点，插件实体导入、二次建表、启动资源安装、 多 worker ready barrier 和生命周期钩子等细节留在插件运行时内部。 """ def __init__( self, startup_manager: PluginRuntimeStartupManager | None = None, *, ready_key: str = LockConstant.PLUGIN_STARTUP_READY_KEY, ready_expire_seconds: int = LockConstant.PLUGIN_STARTUP_READY_EXPIRE_SECONDS, failed_expire_seconds: int = LockConstant.PLUGIN_STARTUP_FAILED_EXPIRE_SECONDS, ready_wait_timeout_seconds: int = LockConstant.PLUGIN_STARTUP_READY_WAIT_TIMEOUT_SECONDS, ready_wait_interval_seconds: int = LockConstant.PLUGIN_STARTUP_READY_WAIT_INTERVAL_SECONDS, lifecycle_lock: PluginLifecycleLock | None = None, startup_generation: str | None = None, release_worker: object | None = None, ) -> None: """ 初始化插件应用运行时适配器。 :param startup_manager: 插件启动协调器 :param ready_key: 插件启动 ready 标记 key :param ready_expire_seconds: ready 标记过期时间 :param failed_expire_seconds: 失败标记过期时间 :param ready_wait_timeout_seconds: 等待 ready 超时时间 :param ready_wait_interval_seconds: 等待 ready 轮询间隔 :param lifecycle_lock: 插件生命周期全局锁 :param startup_generation: 测试或部署显式注入的启动代际 :param release_worker: 维护当前发布快照和 worker 心跳的可选协调器 :return: None """ self.startup_manager = startup_manager or PluginRuntimeStartupManager() self.ready_key = ready_key self.ready_expire_seconds = ready_expire_seconds self.failed_expire_seconds = failed_expire_seconds self.ready_wait_timeout_seconds = ready_wait_timeout_seconds self.ready_wait_interval_seconds = ready_wait_interval_seconds self.lifecycle_lock = lifecycle_lock or NoopPluginLifecycleLock() self.startup_generation = startup_generation self.release_worker = release_worker def bind_app(self, app: FastAPI) -> None: """ 绑定插件运行时到 FastAPI app。 :param app: FastAPI对象 :return: None """ app.state.plugin_application_runtime = self self.startup_manager.bind_app(app) def prepare_metadata(self, app: FastAPI) -> None: """ 准备插件平台自身元数据。 :param app: FastAPI对象 :return: None """ self._ensure_bound(app) self.startup_manager.import_builtin_entities() if self.release_worker is not None: from plugins.core.management.entity.do import release_models # noqa: F401, PLC0415 async def startup( self, app: FastAPI, *, create_tables: Callable[[], Awaitable[None]], ) -> None: """ 启动插件运行时。 同一发布代际只有一个 worker 能在全局生命周期锁内执行写入；其他 worker 等待 当前代际 ready 后仅执行进程本地激活。插件启动不再依赖长期持有的调度器主节点锁。 :param app: FastAPI对象 :param create_tables: 数据库建表回调 :return: None """ try: if self.release_worker is not None: await self.release_worker.start( app, self.startup_manager.builder, self.lifecycle_lock, timeout=self.ready_wait_timeout_seconds ) self.startup_generation = self.release_worker.generation(self.resolve_startup_generation()) app.state.plugin_release_worker = self.release_worker await self._startup_coordinated(app, create_tables=create_tables) if self.release_worker is not None: await self.release_worker.ready() runtime = getattr(app.state, 'plugin_explicit_runtime', None) if isinstance(runtime, ExplicitPluginRuntime): if self.release_worker is not None: # 即使全部显式插件准备失败，空运行时也属于已有发布 worker。 runtime.metrics.worker_id = self.release_worker.worker_id runtime.metrics_reporter.start(getattr(app.state, 'redis', None), f'{self.ready_key}:metrics') reporter = PluginDiagnosticsReporter(runtime.diagnostic_snapshot, runtime.metrics.worker_id) app.state.plugin_diagnostics_reporter = reporter reporter.start(getattr(app.state, 'redis', None), diagnostics_namespace(self.ready_key)) except BaseException: # 激活后的 ready 发布或锁释放也可能失败，此时 server 尚未标记启动完成。 # 显式资源需在这里释放，不能依赖正常 shutdown 分支。 runtime = getattr(app.state, 'plugin_explicit_runtime', None) try: if isinstance(runtime, ExplicitPluginRuntime): await runtime.shutdown() finally: await self._stop_diagnostics(app) if self.release_worker is not None: try: await self.release_worker.stop(failed=True) except Exception: logger.exception('插件启动失败后的 worker 状态上报失败；旧心跳将按 TTL 失效') raise async def _startup_coordinated( self, app: FastAPI, *, create_tables: Callable[[], Awaitable[None]], ) -> None: """ 执行 writer/reader 协调，包括激活后的 ready 发布及锁释放。 :param app: 宿主 FastAPI 应用 :param create_tables: 在允许写入的 worker 中创建数据库表的异步回调 :return: None """ self._ensure_bound(app) if self.release_worker is not None: await self.release_worker.assert_snapshot_current() generation = self.resolve_startup_generation() app.state.plugin_startup_generation = generation app.state.plugin_startup_write_enabled = False ready_state = await self.get_startup_state(app, generation) logger.bind( startup_generation=generation, plugin_startup_role='reader', ready_status=ready_state.get('status', 'missing'), ).debug('🔎 开始检查插件启动代际 ready 状态') activated, stale_ready_reported = await self._process_startup_state( app, generation, ready_state, stale_ready_reported=False, ready_message='复用当前代际插件 ready，开始本地激活', failed_message='检测到当前代际插件启动 failed marker', ) if activated: return deadline = time.monotonic() + self.ready_wait_timeout_seconds wait_reported = False while time.monotonic() < deadline: async with self.lifecycle_lock.lock('__runtime__', f'startup:{generation}') as lock_result: if lock_result.acquired: ready_state = await self.get_startup_state(app, generation) activated, stale_ready_reported = await self._process_startup_state( app, generation, ready_state, stale_ready_reported=stale_ready_reported, ready_message='锁内复用当前代际插件 ready，开始本地激活', failed_message='锁内检测到当前代际插件启动 failed marker', ) if activated: return logger.bind( startup_generation=generation, plugin_startup_role='writer', ready_status=ready_state.get('status', 'missing'), ).info('🎯 当前 worker 成为插件启动 writer') await self._run_startup_writer(app, generation, create_tables) return ready_state = await self.get_startup_state(app, generation) activated, stale_ready_reported = await self._process_startup_state( app, generation, ready_state, stale_ready_reported=stale_ready_reported, ready_message='等待结束，复用当前代际插件 ready', failed_message='等待期间检测到当前代际插件启动 failed marker', ) if activated: return if not wait_reported: logger.bind( startup_generation=generation, plugin_startup_role='reader', ready_status=ready_state.get('status', 'missing'), ).debug('⏳ 等待插件启动 writer 发布当前代际 ready') wait_reported = True await asyncio.sleep(self.ready_wait_interval_seconds) logger.bind( startup_generation=generation, plugin_startup_role='reader', ready_status=ready_state.get('status', 'missing'), ).error('❌ 等待插件启动 ready 超时') raise TimeoutError(f'等待插件启动代际 {generation} ready 超时') async def _process_startup_state( self, app: FastAPI, generation: str, ready_state: Mapping[str, str], *, stale_ready_reported: bool, ready_message: str, failed_message: str, ) -> tuple[bool, bool]: """ 处理一次插件启动状态读取结果。 :param app: FastAPI对象 :param generation: 插件启动代际 :param ready_state: 当前ready状态 :param stale_ready_reported: 是否已记录过期ready告警 :param ready_message: 复用ready时的日志 :param failed_message: 发现failed marker时的日志 :return: 是否已完成reader激活、是否已记录过期ready告警 """ ready_status = ready_state.get('status', 'missing') reader_logger = logger.bind( startup_generation=generation, plugin_startup_role='reader', ready_status=ready_status, ) if ready_status == 'success': if await self.requires_startup_write(): if not stale_ready_reported: logger.bind( startup_generation=generation, plugin_startup_role='reader', ready_status=ready_status, stale_ready_ignored=True, startup_write_required=True, startup_write_reason='missing_default_plugin_state', ).warning('⚠️ 数据库默认插件状态缺失，忽略当前代际旧 ready 标记') return False, True logger.bind( startup_generation=generation, plugin_startup_role='reader', ready_status=ready_status, plugin_install_lifecycle='skipped', plugin_resource_sync='skipped', plugin_entity_table_sync='skipped', ).info( '⏭️ 复用插件 ready 状态，跳过启动期全局写入：' 'plugin_install_lifecycle=skipped，' 'plugin_resource_sync=skipped，' 'plugin_entity_table_sync=skipped' ) reader_logger.debug(f'🔄 {ready_message}') await self._activate_startup_reader(app, generation) return True, stale_ready_reported if ready_status == 'failed': reader_logger.error(f'❌ {failed_message}') raise RuntimeError(self._build_startup_failure_message(ready_state)) return False, stale_ready_reported async def requires_startup_write(self) -> bool: """ 校验 ready 标记对应的数据库状态是否仍然完整。 测试替身或旧适配器未实现该能力时保持原有行为；生产启动管理器会检查 默认启用插件是否仍具备安装状态。 :return: 是否必须重新执行启动期写入 """ checker = getattr(type(self.startup_manager), 'requires_startup_write', None) if checker is None: return False return bool(await self.startup_manager.requires_startup_write()) async def _run_startup_writer( self, app: FastAPI, generation: str, create_tables: Callable[[], Awaitable[None]], ) -> None: """ 执行当前代际唯一的插件启动写入。 :param app: FastAPI对象 :param generation: 插件启动代际 :param create_tables: 数据库建表回调 :return: None """ app.state.plugin_startup_write_enabled = True writer_logger = logger.bind( startup_generation=generation, plugin_startup_role='writer', ready_status='initializing', ) writer_logger.info('🔄 开始同步插件全局启动资源') await self.clear_startup_ready(app, generation) try: await self.startup_manager.prepare_enabled_plugins(app, startup_write_enabled=True) await create_tables() await self.startup_manager.activate_enabled_plugins(app, startup_write_enabled=True) except Exception as exc: await self.mark_startup_failed(app, generation, exc) logger.bind( startup_generation=generation, plugin_startup_role='writer', ready_status='failed', ).exception('❌ 插件全局启动资源同步失败，已写入 failed marker') raise await self.mark_startup_ready(app, generation) logger.bind( startup_generation=generation, plugin_startup_role='writer', ready_status='success', ).info('✅ 插件全局启动资源同步完成') self._log_startup_completed(app, generation, role='writer') async def _activate_startup_reader(self, app: FastAPI, generation: str) -> None: """ 在当前 worker 执行无全局写入的本地插件激活。 :param app: FastAPI对象 :param generation: 插件启动代际 :return: None """ reader_logger = logger.bind( startup_generation=generation, plugin_startup_role='reader', ready_status='success', ) reader_logger.debug('🔄 开始执行插件本地实体、Hook和路由激活') await self.startup_manager.prepare_enabled_plugins(app, startup_write_enabled=False) await self.startup_manager.activate_enabled_plugins(app, startup_write_enabled=False) reader_logger.debug('✅ 插件本地实体、Hook和路由激活完成') self._log_startup_completed(app, generation, role='reader') @staticmethod def _log_startup_completed(app: FastAPI, generation: str, *, role: str) -> None: """ 为每个成功启动的worker输出一条稳定可见的插件运行时摘要。 writer和reader内部过程仍按原有级别记录；该INFO摘要用于避免命中ready后 只有DEBUG日志而表现为插件日志随机缺失。 :param app: FastAPI对象 :param generation: 插件启动代际 :param role: 当前worker的插件启动角色 :return: None """ plugin_registry = getattr(app.state, 'plugin_registry', None) enabled_plugin_ids = ( sorted(str(plugin.plugin_id) for plugin in plugin_registry.list_enabled_plugins()) if plugin_registry is not None else [] ) enabled_plugins = ','.join(enabled_plugin_ids) or 'none' logger.bind( startup_generation=generation, plugin_startup_role=role, ready_status='success', enabled_plugin_ids=enabled_plugin_ids, enabled_plugin_count=len(enabled_plugin_ids), ).info(f'✅ 插件运行时启动完成：role={role}，generation={generation[:8]}，enabled={enabled_plugins}') async def shutdown(self, app: FastAPI) -> None: """ 关闭插件运行时。 :param app: FastAPI对象 :return: None """ self._ensure_bound(app) startup_write_enabled = bool(getattr(app.state, 'plugin_startup_write_enabled', False)) try: await self.startup_manager.shutdown(app, startup_write_enabled=startup_write_enabled) finally: await self._stop_diagnostics(app) if self.release_worker is not None: await self.release_worker.stop() @staticmethod async def _stop_diagnostics(app: FastAPI) -> None: """ 关闭诊断采样，观测清理失败不阻止发布 worker 上报最终状态。 :param app: 宿主 FastAPI 应用 :return: None """ reporter = getattr(app.state, 'plugin_diagnostics_reporter', None) if isinstance(reporter, PluginDiagnosticsReporter): try: await reporter.stop() except Exception: logger.warning('插件运行诊断上报关闭失败，远程快照将按 TTL 过期') async def clear_startup_ready(self, app: FastAPI, generation: str | None = None) -> None: """ 清除指定代际的插件启动 ready 标记。 :param app: FastAPI对象 :param generation: 插件启动代际 :return: None """ resolved_generation = generation or self.resolve_startup_generation() await app.state.redis.delete(self.build_ready_key(resolved_generation)) async def mark_startup_ready(self, app: FastAPI, generation: str | None = None) -> None: """ 标记指定代际已完成插件启动写入。 :param app: FastAPI对象 :param generation: 插件启动代际 :return: None """ resolved_generation = generation or self.resolve_startup_generation() await app.state.redis.set( self.build_ready_key(resolved_generation), json.dumps({'generation': resolved_generation, 'status': 'success'}, ensure_ascii=False), ex=self.ready_expire_seconds, ) writer_logger = logger.bind( startup_generation=resolved_generation, plugin_startup_role='writer', ready_status='success', ) failed_plugin_ids = getattr(app.state, 'plugin_dependency_failed_plugin_ids', set()) if failed_plugin_ids: writer_logger.warning( f'⚠️ 插件启动协调已完成，依赖检查失败插件已隔离：{"、".join(sorted(failed_plugin_ids))}' ) return writer_logger.info('✅ 插件启动资源已就绪') async def mark_startup_failed(self, app: FastAPI, generation: str, error: Exception) -> None: """ 标记指定插件启动代际初始化失败。 :param app: FastAPI对象 :param generation: 插件启动代际 :param error: 启动异常 :return: None """ await app.state.redis.set( self.build_ready_key(generation), json.dumps( { 'generation': generation, 'status': 'failed', 'error': str(error)[:1000], }, ensure_ascii=False, ), ex=self.failed_expire_seconds, ) async def get_startup_state(self, app: FastAPI, generation: str) -> dict[str, str]: """ 读取指定代际的插件启动状态。 :param app: FastAPI对象 :param generation: 插件启动代际 :return: 启动状态 """ raw_state = await app.state.redis.get(self.build_ready_key(generation)) if not raw_state: return {} try: state = json.loads(raw_state) except (TypeError, json.JSONDecodeError): return {} if not isinstance(state, dict) or state.get('generation') != generation: return {} return {str(key): str(value) for key, value in state.items()} def resolve_startup_generation(self) -> str: """ 解析当前插件启动代际。 :return: 插件启动代际 """ if self.startup_generation: return self.startup_generation generation_resolver = PluginStartupGenerationResolver(self.startup_manager.builder.backend_root) self.startup_generation = generation_resolver.resolve() return self.startup_generation def build_ready_key(self, generation: str) -> str: """ 构建代际隔离的 ready key。 :param generation: 插件启动代际 :return: Redis key """ return f'{self.ready_key}:{generation}' @staticmethod def _build_startup_failure_message(state: Mapping[str, str]) -> str: """ 构建启动失败错误消息。 :param state: 启动状态 :return: 错误消息 """ error_message = state.get('error') or '未知错误' return f'插件启动代际 {state.get("generation", "")} 初始化失败：{error_message}' def _ensure_bound(self, app: FastAPI) -> None: """ 确保插件运行时已绑定到 app。 :param app: FastAPI对象 :return: None """ if getattr(app.state, 'plugin_application_runtime', None) is not self: self.bind_app(app) @cache def get_plugin_application_runtime() -> PluginApplicationRuntime: """ 获取应用插件运行时适配器。 :return: 应用插件运行时适配器 """ from plugins.core.deployment.config import PluginDeploymentConfig # noqa: PLC0415 from plugins.core.management.service.startup_gateway import ( # noqa: PLC0415 PluginManagementRouteStateGateway, PluginManagementStartupGateway, ) from plugins.core.runtime.service.lifecycle_lock import RedisPluginLifecycleLock # noqa: PLC0415 startup_manager = PluginRuntimeStartupManager( management_gateway=PluginManagementStartupGateway(), route_state_gateway=PluginManagementRouteStateGateway(), ) deployment_config = PluginDeploymentConfig.from_settings(startup_manager.builder.backend_root) release_worker = None if deployment_config.enabled: from plugins.core.deployment.worker import PluginReleaseWorker # noqa: PLC0415 release_worker = PluginReleaseWorker(deployment_config) return PluginApplicationRuntime( startup_manager=startup_manager, lifecycle_lock=RedisPluginLifecycleLock(), release_worker=release_worker, )