“运维与审计控制台”。
主要能力： SuperAdmin 专用入口：组织与权限 → 运维与审计。 操作日志分页查询，支持时间、账号、路径、方法和状态码筛选。 汇总自动排课、课表发布、补考安排三类失败后台任务。 实时检查数据库、缓存、任务通道及积压状态。 聚合 5xx、失败/重试任务、健康探针和备份时效告警。 SQLite 在线备份；MySQL 调用原生客户端备份。 SHA-256 校验及隔离数据库恢复演练，不覆盖业务库。 MySQL 强制使用独立运维连接，容器增加持久化备份卷与数据库客户端。
This commit is contained in:
@@ -0,0 +1,584 @@
|
||||
using System.Diagnostics;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using Jiaowu.Api.Infrastructure.Persistence;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using MySql.Data.MySqlClient;
|
||||
|
||||
namespace Jiaowu.Api.Infrastructure.Operations;
|
||||
|
||||
public sealed record BackupArtifact(
|
||||
string Id,
|
||||
string FileName,
|
||||
string Provider,
|
||||
DateTime CreatedAt,
|
||||
long SizeBytes,
|
||||
string Sha256,
|
||||
string? Note,
|
||||
DateTime? LastDrillAt,
|
||||
bool? LastDrillSucceeded,
|
||||
string? LastDrillDetail,
|
||||
long? LastDrillDurationMilliseconds);
|
||||
|
||||
public sealed record RestoreDrillResult(
|
||||
string BackupId,
|
||||
bool Succeeded,
|
||||
DateTime CompletedAt,
|
||||
string Detail,
|
||||
long DurationMilliseconds,
|
||||
int? TableCount);
|
||||
|
||||
public sealed class DatabaseBackupService(
|
||||
DatabaseOptions databaseOptions,
|
||||
OperationsOptions options,
|
||||
IConfiguration configuration,
|
||||
IHostEnvironment environment,
|
||||
ILogger<DatabaseBackupService> logger)
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = true
|
||||
};
|
||||
|
||||
private readonly SemaphoreSlim operationLock = new(1, 1);
|
||||
private readonly string backupDirectory = ResolveBackupDirectory(
|
||||
options.BackupDirectory,
|
||||
environment.ContentRootPath);
|
||||
|
||||
public async Task<IReadOnlyCollection<BackupArtifact>> ListAsync(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Directory.CreateDirectory(backupDirectory);
|
||||
var items = new List<BackupArtifact>();
|
||||
foreach (var metadataPath in Directory.EnumerateFiles(
|
||||
backupDirectory,
|
||||
"*.metadata.json",
|
||||
SearchOption.TopDirectoryOnly))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
await using var stream = File.OpenRead(metadataPath);
|
||||
var artifact = await JsonSerializer.DeserializeAsync<BackupArtifact>(
|
||||
stream,
|
||||
JsonOptions,
|
||||
cancellationToken);
|
||||
if (artifact is not null &&
|
||||
File.Exists(Path.Combine(backupDirectory, artifact.FileName)))
|
||||
{
|
||||
items.Add(artifact);
|
||||
}
|
||||
}
|
||||
catch (Exception exception) when (
|
||||
exception is IOException or UnauthorizedAccessException or JsonException)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Unable to read backup metadata {MetadataFile}.",
|
||||
Path.GetFileName(metadataPath));
|
||||
}
|
||||
}
|
||||
|
||||
return items
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
public async Task<BackupArtifact> CreateAsync(
|
||||
string? note,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await operationLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(backupDirectory);
|
||||
var createdAt = DateTime.UtcNow;
|
||||
var id = $"{createdAt:yyyyMMddHHmmss}-{Guid.NewGuid():N}"[..29];
|
||||
var provider = NormalizeProvider(databaseOptions.Provider);
|
||||
var extension = provider == "SQLite" ? ".sqlite" : ".sql";
|
||||
var fileName = $"jiaowu-{id}{extension}";
|
||||
var backupPath = Path.Combine(backupDirectory, fileName);
|
||||
|
||||
try
|
||||
{
|
||||
if (provider == "SQLite")
|
||||
await CreateSqliteBackupAsync(backupPath, cancellationToken);
|
||||
else
|
||||
await CreateMySqlBackupAsync(backupPath, cancellationToken);
|
||||
|
||||
var fileInfo = new FileInfo(backupPath);
|
||||
var artifact = new BackupArtifact(
|
||||
id,
|
||||
fileName,
|
||||
provider,
|
||||
createdAt,
|
||||
fileInfo.Length,
|
||||
await ComputeHashAsync(backupPath, cancellationToken),
|
||||
NormalizeNote(note),
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null);
|
||||
await WriteMetadataAsync(artifact, cancellationToken);
|
||||
return artifact;
|
||||
}
|
||||
catch
|
||||
{
|
||||
if (File.Exists(backupPath))
|
||||
File.Delete(backupPath);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
operationLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<RestoreDrillResult> RunRestoreDrillAsync(
|
||||
string backupId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await operationLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
var artifact = (await ListAsync(cancellationToken))
|
||||
.SingleOrDefault(x => x.Id.Equals(backupId, StringComparison.Ordinal));
|
||||
if (artifact is null)
|
||||
throw new FileNotFoundException("未找到指定备份。");
|
||||
|
||||
var backupPath = Path.Combine(backupDirectory, artifact.FileName);
|
||||
var actualHash = await ComputeHashAsync(backupPath, cancellationToken);
|
||||
if (!CryptographicOperations.FixedTimeEquals(
|
||||
Convert.FromHexString(artifact.Sha256),
|
||||
Convert.FromHexString(actualHash)))
|
||||
{
|
||||
var damaged = await CompleteDrillAsync(
|
||||
artifact,
|
||||
false,
|
||||
"备份文件校验和不一致，恢复演练已中止。",
|
||||
0,
|
||||
null,
|
||||
cancellationToken);
|
||||
return damaged;
|
||||
}
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
RestoreDrillResult result;
|
||||
try
|
||||
{
|
||||
var tableCount = artifact.Provider == "SQLite"
|
||||
? await DrillSqliteAsync(backupPath, cancellationToken)
|
||||
: await DrillMySqlAsync(backupPath, cancellationToken);
|
||||
stopwatch.Stop();
|
||||
result = await CompleteDrillAsync(
|
||||
artifact,
|
||||
true,
|
||||
$"已在隔离数据库完成恢复并通过完整性检查，共发现 {tableCount} 张业务表。",
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
tableCount,
|
||||
cancellationToken);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Restore drill failed for backup {BackupId}.",
|
||||
artifact.Id);
|
||||
result = await CompleteDrillAsync(
|
||||
artifact,
|
||||
false,
|
||||
$"恢复演练失败：{SafeMessage(exception)}",
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
null,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
finally
|
||||
{
|
||||
operationLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CreateSqliteBackupAsync(
|
||||
string backupPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var sourceBuilder = new SqliteConnectionStringBuilder(
|
||||
configuration.GetConnectionString("SQLite")
|
||||
?? throw new InvalidOperationException("缺少 ConnectionStrings:SQLite。"));
|
||||
if (!Path.IsPathRooted(sourceBuilder.DataSource))
|
||||
{
|
||||
sourceBuilder.DataSource = Path.GetFullPath(
|
||||
sourceBuilder.DataSource,
|
||||
environment.ContentRootPath);
|
||||
}
|
||||
|
||||
var destinationBuilder = new SqliteConnectionStringBuilder
|
||||
{
|
||||
DataSource = backupPath,
|
||||
Mode = SqliteOpenMode.ReadWriteCreate,
|
||||
Pooling = false
|
||||
};
|
||||
await using var source = new SqliteConnection(sourceBuilder.ConnectionString);
|
||||
await using var destination = new SqliteConnection(destinationBuilder.ConnectionString);
|
||||
await source.OpenAsync(cancellationToken);
|
||||
await destination.OpenAsync(cancellationToken);
|
||||
source.BackupDatabase(destination);
|
||||
}
|
||||
|
||||
private async Task CreateMySqlBackupAsync(
|
||||
string backupPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var connection = GetMySqlConnectionBuilder();
|
||||
var arguments = new List<string>
|
||||
{
|
||||
"--protocol=tcp",
|
||||
$"--host={connection.Server}",
|
||||
$"--port={connection.Port}",
|
||||
$"--user={connection.UserID}",
|
||||
"--single-transaction",
|
||||
"--quick",
|
||||
"--routines",
|
||||
"--triggers",
|
||||
"--events",
|
||||
"--hex-blob",
|
||||
"--default-character-set=utf8mb4"
|
||||
};
|
||||
arguments.AddRange(options.MySqlAdditionalArguments);
|
||||
arguments.Add(connection.Database);
|
||||
|
||||
await using var output = new FileStream(
|
||||
backupPath,
|
||||
FileMode.CreateNew,
|
||||
FileAccess.Write,
|
||||
FileShare.None,
|
||||
81920,
|
||||
FileOptions.Asynchronous);
|
||||
await RunToolAsync(
|
||||
options.MySqlDumpPath,
|
||||
arguments,
|
||||
connection.Password,
|
||||
standardInput: null,
|
||||
standardOutput: output,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task<int> DrillSqliteAsync(
|
||||
string backupPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var drillPath = Path.Combine(
|
||||
Path.GetDirectoryName(backupPath)!,
|
||||
$".restore-drill-{Guid.NewGuid():N}.sqlite");
|
||||
try
|
||||
{
|
||||
var sourceBuilder = new SqliteConnectionStringBuilder
|
||||
{
|
||||
DataSource = backupPath,
|
||||
Mode = SqliteOpenMode.ReadOnly,
|
||||
Pooling = false
|
||||
};
|
||||
var drillBuilder = new SqliteConnectionStringBuilder
|
||||
{
|
||||
DataSource = drillPath,
|
||||
Mode = SqliteOpenMode.ReadWriteCreate,
|
||||
Pooling = false
|
||||
};
|
||||
await using var source = new SqliteConnection(sourceBuilder.ConnectionString);
|
||||
await using var drill = new SqliteConnection(drillBuilder.ConnectionString);
|
||||
await source.OpenAsync(cancellationToken);
|
||||
await drill.OpenAsync(cancellationToken);
|
||||
source.BackupDatabase(drill);
|
||||
|
||||
await using var integrity = drill.CreateCommand();
|
||||
integrity.CommandText = "PRAGMA integrity_check;";
|
||||
var integrityResult = Convert.ToString(
|
||||
await integrity.ExecuteScalarAsync(cancellationToken));
|
||||
if (!string.Equals(integrityResult, "ok", StringComparison.OrdinalIgnoreCase))
|
||||
throw new InvalidDataException($"SQLite 完整性检查返回 {integrityResult ?? "空结果"}。");
|
||||
|
||||
await using var tables = drill.CreateCommand();
|
||||
tables.CommandText =
|
||||
"SELECT COUNT(*) FROM sqlite_master " +
|
||||
"WHERE type = 'table' AND name NOT LIKE 'sqlite_%';";
|
||||
return Convert.ToInt32(await tables.ExecuteScalarAsync(cancellationToken));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(drillPath))
|
||||
File.Delete(drillPath);
|
||||
if (File.Exists($"{drillPath}-shm"))
|
||||
File.Delete($"{drillPath}-shm");
|
||||
if (File.Exists($"{drillPath}-wal"))
|
||||
File.Delete($"{drillPath}-wal");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<int> DrillMySqlAsync(
|
||||
string backupPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var connection = GetMySqlConnectionBuilder();
|
||||
var drillDatabase = $"jiaowu_restore_drill_{DateTime.UtcNow:yyyyMMddHHmmss}_" +
|
||||
Guid.NewGuid().ToString("N")[..8];
|
||||
var adminBuilder = new MySqlConnectionStringBuilder(connection.ConnectionString)
|
||||
{
|
||||
Database = ""
|
||||
};
|
||||
|
||||
await using var admin = new MySqlConnection(adminBuilder.ConnectionString);
|
||||
await admin.OpenAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
await using (var create = admin.CreateCommand())
|
||||
{
|
||||
create.CommandText =
|
||||
$"CREATE DATABASE `{drillDatabase}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;";
|
||||
await create.ExecuteNonQueryAsync(cancellationToken);
|
||||
}
|
||||
|
||||
var arguments = new List<string>
|
||||
{
|
||||
"--protocol=tcp",
|
||||
$"--host={connection.Server}",
|
||||
$"--port={connection.Port}",
|
||||
$"--user={connection.UserID}",
|
||||
"--default-character-set=utf8mb4"
|
||||
};
|
||||
arguments.AddRange(options.MySqlAdditionalArguments);
|
||||
arguments.Add($"--database={drillDatabase}");
|
||||
await using (var input = new FileStream(
|
||||
backupPath,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read,
|
||||
81920,
|
||||
FileOptions.Asynchronous))
|
||||
{
|
||||
await RunToolAsync(
|
||||
options.MySqlClientPath,
|
||||
arguments,
|
||||
connection.Password,
|
||||
input,
|
||||
standardOutput: null,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
await using var count = admin.CreateCommand();
|
||||
count.CommandText =
|
||||
"SELECT COUNT(*) FROM information_schema.tables " +
|
||||
"WHERE table_schema = @schema AND table_type = 'BASE TABLE';";
|
||||
count.Parameters.AddWithValue("@schema", drillDatabase);
|
||||
return Convert.ToInt32(await count.ExecuteScalarAsync(cancellationToken));
|
||||
}
|
||||
finally
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var drop = admin.CreateCommand();
|
||||
drop.CommandText = $"DROP DATABASE IF EXISTS `{drillDatabase}`;";
|
||||
await drop.ExecuteNonQueryAsync(CancellationToken.None);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogCritical(
|
||||
exception,
|
||||
"Unable to remove isolated restore drill database {DatabaseName}.",
|
||||
drillDatabase);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunToolAsync(
|
||||
string executable,
|
||||
IReadOnlyCollection<string> arguments,
|
||||
string password,
|
||||
Stream? standardInput,
|
||||
Stream? standardOutput,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var startInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = executable,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardError = true,
|
||||
RedirectStandardInput = standardInput is not null,
|
||||
RedirectStandardOutput = standardOutput is not null,
|
||||
CreateNoWindow = true
|
||||
};
|
||||
foreach (var argument in arguments)
|
||||
startInfo.ArgumentList.Add(argument);
|
||||
if (!string.IsNullOrEmpty(password))
|
||||
startInfo.Environment["MYSQL_PWD"] = password;
|
||||
|
||||
using var process = Process.Start(startInfo)
|
||||
?? throw new InvalidOperationException($"无法启动数据库工具 {executable}。");
|
||||
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
timeout.CancelAfter(TimeSpan.FromMinutes(options.ToolTimeoutMinutes));
|
||||
var errorTask = process.StandardError.ReadToEndAsync(timeout.Token);
|
||||
var inputTask = standardInput is null
|
||||
? Task.CompletedTask
|
||||
: CopyInputAsync(standardInput, process.StandardInput.BaseStream, timeout.Token);
|
||||
var outputTask = standardOutput is null
|
||||
? Task.CompletedTask
|
||||
: process.StandardOutput.BaseStream.CopyToAsync(
|
||||
standardOutput,
|
||||
timeout.Token);
|
||||
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(
|
||||
process.WaitForExitAsync(timeout.Token),
|
||||
inputTask,
|
||||
outputTask);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
if (!process.HasExited)
|
||||
process.Kill(entireProcessTree: true);
|
||||
throw;
|
||||
}
|
||||
|
||||
var error = await errorTask;
|
||||
if (process.ExitCode != 0)
|
||||
throw new InvalidOperationException(
|
||||
$"数据库工具执行失败（退出码 {process.ExitCode}）：{TrimToolError(error)}");
|
||||
}
|
||||
|
||||
private static async Task CopyInputAsync(
|
||||
Stream input,
|
||||
Stream processInput,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await input.CopyToAsync(processInput, cancellationToken);
|
||||
await processInput.FlushAsync(cancellationToken);
|
||||
processInput.Close();
|
||||
}
|
||||
|
||||
private MySqlConnectionStringBuilder GetMySqlConnectionBuilder()
|
||||
{
|
||||
var value = configuration.GetConnectionString("OperationsMySql");
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"MySQL 备份与恢复演练必须配置独立的 " +
|
||||
"ConnectionStrings:OperationsMySql 运维账号，不能复用日常业务账号。");
|
||||
}
|
||||
var builder = new MySqlConnectionStringBuilder(value);
|
||||
if (string.IsNullOrWhiteSpace(builder.Database))
|
||||
throw new InvalidOperationException(
|
||||
"OperationsMySql 连接字符串未指定业务数据库名称。");
|
||||
return builder;
|
||||
}
|
||||
|
||||
private async Task<RestoreDrillResult> CompleteDrillAsync(
|
||||
BackupArtifact artifact,
|
||||
bool succeeded,
|
||||
string detail,
|
||||
long durationMilliseconds,
|
||||
int? tableCount,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var completedAt = DateTime.UtcNow;
|
||||
var updated = artifact with
|
||||
{
|
||||
LastDrillAt = completedAt,
|
||||
LastDrillSucceeded = succeeded,
|
||||
LastDrillDetail = detail,
|
||||
LastDrillDurationMilliseconds = durationMilliseconds
|
||||
};
|
||||
await WriteMetadataAsync(updated, cancellationToken);
|
||||
return new RestoreDrillResult(
|
||||
artifact.Id,
|
||||
succeeded,
|
||||
completedAt,
|
||||
detail,
|
||||
durationMilliseconds,
|
||||
tableCount);
|
||||
}
|
||||
|
||||
private async Task WriteMetadataAsync(
|
||||
BackupArtifact artifact,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var metadataPath = Path.Combine(
|
||||
backupDirectory,
|
||||
$"{artifact.Id}.metadata.json");
|
||||
var temporaryPath = $"{metadataPath}.{Guid.NewGuid():N}.tmp";
|
||||
try
|
||||
{
|
||||
await using (var stream = new FileStream(
|
||||
temporaryPath,
|
||||
FileMode.CreateNew,
|
||||
FileAccess.Write,
|
||||
FileShare.None,
|
||||
16384,
|
||||
FileOptions.Asynchronous))
|
||||
{
|
||||
await JsonSerializer.SerializeAsync(
|
||||
stream,
|
||||
artifact,
|
||||
JsonOptions,
|
||||
cancellationToken);
|
||||
}
|
||||
File.Move(temporaryPath, metadataPath, overwrite: true);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(temporaryPath))
|
||||
File.Delete(temporaryPath);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<string> ComputeHashAsync(
|
||||
string path,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var stream = File.OpenRead(path);
|
||||
return Convert.ToHexString(
|
||||
await SHA256.HashDataAsync(stream, cancellationToken));
|
||||
}
|
||||
|
||||
private static string ResolveBackupDirectory(
|
||||
string configuredPath,
|
||||
string contentRoot)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(configuredPath))
|
||||
throw new InvalidOperationException("Operations:BackupDirectory 不能为空。");
|
||||
return Path.IsPathRooted(configuredPath)
|
||||
? Path.GetFullPath(configuredPath)
|
||||
: Path.GetFullPath(configuredPath, contentRoot);
|
||||
}
|
||||
|
||||
private static string NormalizeProvider(string provider) =>
|
||||
provider.Equals("SQLite", StringComparison.OrdinalIgnoreCase)
|
||||
? "SQLite"
|
||||
: provider.Equals("MySql", StringComparison.OrdinalIgnoreCase)
|
||||
? "MySql"
|
||||
: throw new InvalidOperationException($"不支持数据库 Provider '{provider}'。");
|
||||
|
||||
private static string? NormalizeNote(string? note)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(note)) return null;
|
||||
var trimmed = note.Trim();
|
||||
return trimmed.Length <= 200 ? trimmed : trimmed[..200];
|
||||
}
|
||||
|
||||
private static string SafeMessage(Exception exception)
|
||||
{
|
||||
var message = exception.GetBaseException().Message;
|
||||
return message.Length <= 500 ? message : message[..500];
|
||||
}
|
||||
|
||||
private static string TrimToolError(string error)
|
||||
{
|
||||
var trimmed = error.Trim();
|
||||
if (trimmed.Length == 0) return "未返回错误详情。";
|
||||
return trimmed.Length <= 500 ? trimmed : trimmed[..500];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
using System.Diagnostics;
|
||||
using Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||
using Jiaowu.Api.Infrastructure.Caching;
|
||||
using Jiaowu.Api.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Caching.Distributed;
|
||||
|
||||
namespace Jiaowu.Api.Infrastructure.Operations;
|
||||
|
||||
public sealed record OperationalComponentHealth(
|
||||
string Key,
|
||||
string Label,
|
||||
string Status,
|
||||
string Backend,
|
||||
long? LatencyMilliseconds,
|
||||
string Detail);
|
||||
|
||||
public sealed record OperationalHealthSnapshot(
|
||||
DateTime CheckedAt,
|
||||
string OverallStatus,
|
||||
IReadOnlyCollection<OperationalComponentHealth> Components,
|
||||
BackgroundJobBacklogSnapshot? Backlog);
|
||||
|
||||
public sealed class OperationalHealthService(
|
||||
AppDbContext db,
|
||||
IServiceProvider services,
|
||||
IBackgroundJobTransport transport,
|
||||
BackgroundJobMonitoringService monitoring,
|
||||
DatabaseOptions databaseOptions,
|
||||
AppCacheOptions cacheOptions,
|
||||
IConfiguration configuration)
|
||||
{
|
||||
public async Task<OperationalHealthSnapshot> CheckAsync(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var database = await CheckDatabaseAsync(cancellationToken);
|
||||
var cache = await CheckCacheAsync(cancellationToken);
|
||||
var (messaging, backlog) = await CheckMessagingAsync(cancellationToken);
|
||||
var components = new[] { database, cache, messaging };
|
||||
var overall = components.Any(x => x.Status == "unhealthy")
|
||||
? "unhealthy"
|
||||
: components.Any(x => x.Status == "warning")
|
||||
? "warning"
|
||||
: "healthy";
|
||||
return new OperationalHealthSnapshot(
|
||||
DateTime.UtcNow,
|
||||
overall,
|
||||
components,
|
||||
backlog);
|
||||
}
|
||||
|
||||
private async Task<OperationalComponentHealth> CheckDatabaseAsync(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
var canConnect = await db.Database.CanConnectAsync(cancellationToken);
|
||||
stopwatch.Stop();
|
||||
return canConnect
|
||||
? new OperationalComponentHealth(
|
||||
"database",
|
||||
"数据库",
|
||||
"healthy",
|
||||
databaseOptions.Provider,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
"连接与基础查询正常。")
|
||||
: new OperationalComponentHealth(
|
||||
"database",
|
||||
"数据库",
|
||||
"unhealthy",
|
||||
databaseOptions.Provider,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
"无法建立数据库连接。");
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
return new OperationalComponentHealth(
|
||||
"database",
|
||||
"数据库",
|
||||
"unhealthy",
|
||||
databaseOptions.Provider,
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
SafeMessage(exception));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<OperationalComponentHealth> CheckCacheAsync(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!cacheOptions.Enabled)
|
||||
{
|
||||
return new OperationalComponentHealth(
|
||||
"cache",
|
||||
"缓存",
|
||||
"warning",
|
||||
"disabled",
|
||||
null,
|
||||
"缓存已通过配置关闭，所有查询将直接访问数据源。");
|
||||
}
|
||||
|
||||
var hasRedisConfiguration = !string.IsNullOrWhiteSpace(
|
||||
configuration.GetConnectionString("Redis"));
|
||||
var distributedCache = services.GetService<IDistributedCache>();
|
||||
if (!hasRedisConfiguration || distributedCache is null)
|
||||
{
|
||||
return new OperationalComponentHealth(
|
||||
"cache",
|
||||
"缓存",
|
||||
"healthy",
|
||||
"memory",
|
||||
null,
|
||||
"使用进程内混合缓存；服务重启后缓存会自然重建。");
|
||||
}
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
await distributedCache.GetAsync(
|
||||
"jiaowu:operations:health-probe",
|
||||
cancellationToken);
|
||||
stopwatch.Stop();
|
||||
return new OperationalComponentHealth(
|
||||
"cache",
|
||||
"缓存",
|
||||
"healthy",
|
||||
"redis",
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
"Redis 连接与读取探针正常。");
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
return new OperationalComponentHealth(
|
||||
"cache",
|
||||
"缓存",
|
||||
"unhealthy",
|
||||
"redis",
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
SafeMessage(exception));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<(OperationalComponentHealth Component,
|
||||
BackgroundJobBacklogSnapshot? Backlog)> CheckMessagingAsync(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
BackgroundJobBacklogSnapshot? backlog = null;
|
||||
try
|
||||
{
|
||||
var healthy = await transport.CheckHealthAsync(cancellationToken);
|
||||
backlog = await monitoring.GetSnapshotAsync(cancellationToken);
|
||||
stopwatch.Stop();
|
||||
var stuck = backlog.ExpiredLeases > 0 ||
|
||||
backlog.OldestUnfinishedAgeSeconds > 1800;
|
||||
var status = !healthy
|
||||
? "unhealthy"
|
||||
: stuck
|
||||
? "warning"
|
||||
: "healthy";
|
||||
var detail = !healthy
|
||||
? "后台任务传输不可用。"
|
||||
: stuck
|
||||
? $"发现 {backlog.ExpiredLeases} 个过期租约，最早未完成任务已等待 " +
|
||||
$"{Math.Round(backlog.OldestUnfinishedAgeSeconds ?? 0)} 秒。"
|
||||
: $"待发布 {backlog.Pending + backlog.Publishing}，" +
|
||||
$"待处理 {backlog.Published + backlog.Processing}。";
|
||||
return (
|
||||
new OperationalComponentHealth(
|
||||
"messaging",
|
||||
"后台任务通道",
|
||||
status,
|
||||
transport.IsDurable ? "rabbitmq" : "memory",
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
detail),
|
||||
backlog);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
return (
|
||||
new OperationalComponentHealth(
|
||||
"messaging",
|
||||
"后台任务通道",
|
||||
"unhealthy",
|
||||
transport.IsDurable ? "rabbitmq" : "memory",
|
||||
stopwatch.ElapsedMilliseconds,
|
||||
SafeMessage(exception)),
|
||||
backlog);
|
||||
}
|
||||
}
|
||||
|
||||
private static string SafeMessage(Exception exception)
|
||||
{
|
||||
var message = exception.GetBaseException().Message;
|
||||
return message.Length <= 300 ? message : message[..300];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
namespace Jiaowu.Api.Infrastructure.Operations;
|
||||
|
||||
public sealed class OperationsOptions
|
||||
{
|
||||
public const string SectionName = "Operations";
|
||||
|
||||
public string BackupDirectory { get; set; } = "data/backups";
|
||||
public int BackupWarningHours { get; set; } = 24;
|
||||
public int ToolTimeoutMinutes { get; set; } = 30;
|
||||
public string MySqlDumpPath { get; set; } = "mysqldump";
|
||||
public string MySqlClientPath { get; set; } = "mysql";
|
||||
public string[] MySqlAdditionalArguments { get; set; } = [];
|
||||
}
|
||||
Reference in New Issue
Block a user