“运维与审计控制台”。
主要能力： SuperAdmin 专用入口：组织与权限 → 运维与审计。 操作日志分页查询，支持时间、账号、路径、方法和状态码筛选。 汇总自动排课、课表发布、补考安排三类失败后台任务。 实时检查数据库、缓存、任务通道及积压状态。 聚合 5xx、失败/重试任务、健康探针和备份时效告警。 SQLite 在线备份；MySQL 调用原生客户端备份。 SHA-256 校验及隔离数据库恢复演练，不覆盖业务库。 MySQL 强制使用独立运维连接，容器增加持久化备份卷与数据库客户端。
This commit is contained in:
@@ -0,0 +1,200 @@
|
||||
using System.Diagnostics;
|
||||
using Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||
using Jiaowu.Api.Infrastructure.Caching;
|
||||
using Jiaowu.Api.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Caching.Distributed;
|
||||
|
||||
namespace Jiaowu.Api.Infrastructure.Operations;
|
||||
|
||||
public sealed record OperationalComponentHealth(
|
||||
string Key,
|
||||
string Label,
|
||||
string Status,
|
||||
string Backend,
|
||||
long? LatencyMilliseconds,
|
||||
string Detail);
|
||||
|
||||
public sealed record OperationalHealthSnapshot(
|
||||
DateTime CheckedAt,
|
||||
string OverallStatus,
|
||||
IReadOnlyCollection<OperationalComponentHealth> Components,
|
||||
BackgroundJobBacklogSnapshot? Backlog);
|
||||
|
||||
public sealed class OperationalHealthService(
|
||||
AppDbContext db,
|
||||
IServiceProvider services,
|
||||
IBackgroundJobTransport transport,
|
||||
BackgroundJobMonitoringService monitoring,
|
||||
DatabaseOptions databaseOptions,
|
||||
AppCacheOptions cacheOptions,
|
||||
IConfiguration configuration)
|
||||
{
|
||||
public async Task<OperationalHealthSnapshot> CheckAsync(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var database = await CheckDatabaseAsync(cancellationToken);
|
||||
var cache = await CheckCacheAsync(cancellationToken);
|
||||
var (messaging, backlog) = await CheckMessagingAsync(cancellationToken);
|
||||
var components = new[] { database, cache, messaging };
|
||||
var overall = components.Any(x => x.Status == "unhealthy")
|
||||
? "unhealthy"
|
||||
: components.Any(x => x.Status == "warning")
|
||||
? "warning"
|
||||
: "healthy";
|
||||
return new OperationalHealthSnapshot(
|
||||
DateTime.UtcNow,
|
||||
overall,
|
||||
components,
|
||||
backlog);
|
||||
}
|
||||
|
||||
private async Task<OperationalComponentHealth> CheckDatabaseAsync(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
var canConnect = await db.Database.CanConnectAsync(cancellationToken);
|
||||
stopwatch.Stop();
|
||||
return canConnect
|
||||
? new OperationalComponentHealth(
|
||||
"database",
|
||||
"数据库",
|
||||
"healthy",
|
||||
databaseOptions.Provider,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
"连接与基础查询正常。")
|
||||
: new OperationalComponentHealth(
|
||||
"database",
|
||||
"数据库",
|
||||
"unhealthy",
|
||||
databaseOptions.Provider,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
"无法建立数据库连接。");
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
return new OperationalComponentHealth(
|
||||
"database",
|
||||
"数据库",
|
||||
"unhealthy",
|
||||
databaseOptions.Provider,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
SafeMessage(exception));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<OperationalComponentHealth> CheckCacheAsync(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!cacheOptions.Enabled)
|
||||
{
|
||||
return new OperationalComponentHealth(
|
||||
"cache",
|
||||
"缓存",
|
||||
"warning",
|
||||
"disabled",
|
||||
null,
|
||||
"缓存已通过配置关闭，所有查询将直接访问数据源。");
|
||||
}
|
||||
|
||||
var hasRedisConfiguration = !string.IsNullOrWhiteSpace(
|
||||
configuration.GetConnectionString("Redis"));
|
||||
var distributedCache = services.GetService<IDistributedCache>();
|
||||
if (!hasRedisConfiguration || distributedCache is null)
|
||||
{
|
||||
return new OperationalComponentHealth(
|
||||
"cache",
|
||||
"缓存",
|
||||
"healthy",
|
||||
"memory",
|
||||
null,
|
||||
"使用进程内混合缓存；服务重启后缓存会自然重建。");
|
||||
}
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
await distributedCache.GetAsync(
|
||||
"jiaowu:operations:health-probe",
|
||||
cancellationToken);
|
||||
stopwatch.Stop();
|
||||
return new OperationalComponentHealth(
|
||||
"cache",
|
||||
"缓存",
|
||||
"healthy",
|
||||
"redis",
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
"Redis 连接与读取探针正常。");
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
return new OperationalComponentHealth(
|
||||
"cache",
|
||||
"缓存",
|
||||
"unhealthy",
|
||||
"redis",
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
SafeMessage(exception));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<(OperationalComponentHealth Component,
|
||||
BackgroundJobBacklogSnapshot? Backlog)> CheckMessagingAsync(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
BackgroundJobBacklogSnapshot? backlog = null;
|
||||
try
|
||||
{
|
||||
var healthy = await transport.CheckHealthAsync(cancellationToken);
|
||||
backlog = await monitoring.GetSnapshotAsync(cancellationToken);
|
||||
stopwatch.Stop();
|
||||
var stuck = backlog.ExpiredLeases > 0 ||
|
||||
backlog.OldestUnfinishedAgeSeconds > 1800;
|
||||
var status = !healthy
|
||||
? "unhealthy"
|
||||
: stuck
|
||||
? "warning"
|
||||
: "healthy";
|
||||
var detail = !healthy
|
||||
? "后台任务传输不可用。"
|
||||
: stuck
|
||||
? $"发现 {backlog.ExpiredLeases} 个过期租约，最早未完成任务已等待 " +
|
||||
$"{Math.Round(backlog.OldestUnfinishedAgeSeconds ?? 0)} 秒。"
|
||||
: $"待发布 {backlog.Pending + backlog.Publishing}，" +
|
||||
$"待处理 {backlog.Published + backlog.Processing}。";
|
||||
return (
|
||||
new OperationalComponentHealth(
|
||||
"messaging",
|
||||
"后台任务通道",
|
||||
status,
|
||||
transport.IsDurable ? "rabbitmq" : "memory",
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
detail),
|
||||
backlog);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
return (
|
||||
new OperationalComponentHealth(
|
||||
"messaging",
|
||||
"后台任务通道",
|
||||
"unhealthy",
|
||||
transport.IsDurable ? "rabbitmq" : "memory",
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
SafeMessage(exception)),
|
||||
backlog);
|
||||
}
|
||||
}
|
||||
|
||||
private static string SafeMessage(Exception exception)
|
||||
{
|
||||
var message = exception.GetBaseException().Message;
|
||||
return message.Length <= 300 ? message : message[..300];
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user