using System.Diagnostics; using Jiaowu.Api.Infrastructure.BackgroundJobs; using Jiaowu.Api.Infrastructure.Caching; using Jiaowu.Api.Infrastructure.Persistence; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Caching.Distributed; namespace Jiaowu.Api.Infrastructure.Operations; public sealed record OperationalComponentHealth( string Key, string Label, string Status, string Backend, long? LatencyMilliseconds, string Detail); public sealed record OperationalHealthSnapshot( DateTime CheckedAt, string OverallStatus, IReadOnlyCollection Components, BackgroundJobBacklogSnapshot? Backlog); public sealed class OperationalHealthService( AppDbContext db, IServiceProvider services, IBackgroundJobTransport transport, BackgroundJobMonitoringService monitoring, DatabaseOptions databaseOptions, AppCacheOptions cacheOptions, IConfiguration configuration) { public async Task CheckAsync( CancellationToken cancellationToken) { var database = await CheckDatabaseAsync(cancellationToken); var cache = await CheckCacheAsync(cancellationToken); var (messaging, backlog) = await CheckMessagingAsync(cancellationToken); var components = new[] { database, cache, messaging }; var overall = components.Any(x => x.Status == "unhealthy") ? "unhealthy" : components.Any(x => x.Status == "warning") ? "warning" : "healthy"; return new OperationalHealthSnapshot( DateTime.UtcNow, overall, components, backlog); } private async Task CheckDatabaseAsync( CancellationToken cancellationToken) { var stopwatch = Stopwatch.StartNew(); try { var canConnect = await db.Database.CanConnectAsync(cancellationToken); stopwatch.Stop(); return canConnect ? new OperationalComponentHealth( "database", "数据库", "healthy", databaseOptions.Provider, stopwatch.ElapsedMilliseconds, "连接与基础查询正常。") : new OperationalComponentHealth( "database", "数据库", "unhealthy", databaseOptions.Provider, stopwatch.ElapsedMilliseconds, "无法建立数据库连接。"); } catch (Exception exception) when (exception is not OperationCanceledException) { stopwatch.Stop(); return new OperationalComponentHealth( "database", "数据库", "unhealthy", databaseOptions.Provider, stopwatch.ElapsedMilliseconds, SafeMessage(exception)); } } private async Task CheckCacheAsync( CancellationToken cancellationToken) { if (!cacheOptions.Enabled) { return new OperationalComponentHealth( "cache", "缓存", "warning", "disabled", null, "缓存已通过配置关闭,所有查询将直接访问数据源。"); } var hasRedisConfiguration = !string.IsNullOrWhiteSpace( configuration.GetConnectionString("Redis")); var distributedCache = services.GetService(); if (!hasRedisConfiguration || distributedCache is null) { return new OperationalComponentHealth( "cache", "缓存", "healthy", "memory", null, "使用进程内混合缓存;服务重启后缓存会自然重建。"); } var stopwatch = Stopwatch.StartNew(); try { await distributedCache.GetAsync( "jiaowu:operations:health-probe", cancellationToken); stopwatch.Stop(); return new OperationalComponentHealth( "cache", "缓存", "healthy", "redis", stopwatch.ElapsedMilliseconds, "Redis 连接与读取探针正常。"); } catch (Exception exception) when (exception is not OperationCanceledException) { stopwatch.Stop(); return new OperationalComponentHealth( "cache", "缓存", "unhealthy", "redis", stopwatch.ElapsedMilliseconds, SafeMessage(exception)); } } private async Task<(OperationalComponentHealth Component, BackgroundJobBacklogSnapshot? Backlog)> CheckMessagingAsync( CancellationToken cancellationToken) { var stopwatch = Stopwatch.StartNew(); BackgroundJobBacklogSnapshot? backlog = null; try { var healthy = await transport.CheckHealthAsync(cancellationToken); backlog = await monitoring.GetSnapshotAsync(cancellationToken); stopwatch.Stop(); var stuck = backlog.ExpiredLeases > 0 || backlog.OldestUnfinishedAgeSeconds > 1800; var status = !healthy ? "unhealthy" : stuck ? "warning" : "healthy"; var detail = !healthy ? "后台任务传输不可用。" : stuck ? $"发现 {backlog.ExpiredLeases} 个过期租约,最早未完成任务已等待 " + $"{Math.Round(backlog.OldestUnfinishedAgeSeconds ?? 0)} 秒。" : $"待发布 {backlog.Pending + backlog.Publishing}," + $"待处理 {backlog.Published + backlog.Processing}。"; return ( new OperationalComponentHealth( "messaging", "后台任务通道", status, transport.IsDurable ? "rabbitmq" : "memory", stopwatch.ElapsedMilliseconds, detail), backlog); } catch (Exception exception) when (exception is not OperationCanceledException) { stopwatch.Stop(); return ( new OperationalComponentHealth( "messaging", "后台任务通道", "unhealthy", transport.IsDurable ? "rabbitmq" : "memory", stopwatch.ElapsedMilliseconds, SafeMessage(exception)), backlog); } } private static string SafeMessage(Exception exception) { var message = exception.GetBaseException().Message; return message.Length <= 300 ? message : message[..300]; } }