“运维与审计控制台”。

主要能力:
SuperAdmin 专用入口:组织与权限 → 运维与审计。
操作日志分页查询,支持时间、账号、路径、方法和状态码筛选。
汇总自动排课、课表发布、补考安排三类失败后台任务。
实时检查数据库、缓存、任务通道及积压状态。
聚合 5xx、失败/重试任务、健康探针和备份时效告警。
SQLite 在线备份;MySQL 调用原生客户端备份。
SHA-256 校验及隔离数据库恢复演练,不覆盖业务库。
MySQL 强制使用独立运维连接,容器增加持久化备份卷与数据库客户端。
This commit is contained in:
2026-07-27 16:37:45 +08:00 Unverified
parent 14ac49115c
commit 4d8de1e4ae
15 changed files with 2687 additions and 4 deletions
@@ -0,0 +1,519 @@
using System.ComponentModel.DataAnnotations;
using Jiaowu.Api.Contracts;
using Jiaowu.Api.Domain.Academic;
using Jiaowu.Api.Domain.Identity;
using Jiaowu.Api.Domain.System;
using Jiaowu.Api.Infrastructure.Operations;
using Jiaowu.Api.Infrastructure.Persistence;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
namespace Jiaowu.Api.Controllers;
[ApiController]
[Authorize(Roles = SystemRoles.SuperAdmin)]
[Route("api/operations")]
public sealed class OperationsController(
AppDbContext db,
OperationalHealthService healthService,
DatabaseBackupService backupService,
OperationsOptions options) : ControllerBase
{
[HttpGet("summary")]
public async Task<ActionResult<OperationsSummary>> GetSummary(
CancellationToken cancellationToken)
{
var since = DateTime.UtcNow.AddHours(-24);
var health = await healthService.CheckAsync(cancellationToken);
var backups = await backupService.ListAsync(cancellationToken);
var auditCount = await db.AuditLogs.AsNoTracking()
.CountAsync(x => x.CreatedAt >= since, cancellationToken);
var serverErrorCount = await db.AuditLogs.AsNoTracking()
.CountAsync(
x => x.CreatedAt >= since && x.StatusCode >= 500,
cancellationToken);
var failedJobCount = await CountFailedJobsAsync(
DateTime.UtcNow.AddDays(-7),
cancellationToken);
var alerts = await BuildAlertsAsync(
health,
backups,
serverErrorCount,
failedJobCount,
cancellationToken);
return Ok(new OperationsSummary(
DateTime.UtcNow,
health,
new OperationsCounters(
auditCount,
serverErrorCount,
failedJobCount,
alerts.Count(x => x.Severity == "critical")),
alerts,
backups.FirstOrDefault()));
}
[HttpGet("health")]
public async Task<ActionResult<OperationalHealthSnapshot>> GetHealth(
CancellationToken cancellationToken) =>
Ok(await healthService.CheckAsync(cancellationToken));
[HttpGet("audit-logs")]
public async Task<ActionResult<PagedResult<AuditLogItem>>> GetAuditLogs(
[FromQuery] int page = 1,
[FromQuery] int pageSize = 20,
[FromQuery] string? method = null,
[FromQuery] int? statusCode = null,
[FromQuery] string? userName = null,
[FromQuery] string? path = null,
[FromQuery] DateTime? from = null,
[FromQuery] DateTime? to = null,
CancellationToken cancellationToken = default)
{
var pagingError = ValidatePaging(page, pageSize);
if (pagingError is not null) return pagingError;
var rangeError = ValidateRange(from, to);
if (rangeError is not null) return rangeError;
var query = db.AuditLogs.AsNoTracking().AsQueryable();
if (!string.IsNullOrWhiteSpace(method))
{
var normalizedMethod = method.Trim().ToUpperInvariant();
query = query.Where(x => x.Method == normalizedMethod);
}
if (statusCode.HasValue)
query = query.Where(x => x.StatusCode == statusCode.Value);
if (!string.IsNullOrWhiteSpace(userName))
{
var normalizedUser = userName.Trim();
query = query.Where(x =>
x.UserName != null && x.UserName.Contains(normalizedUser));
}
if (!string.IsNullOrWhiteSpace(path))
{
var normalizedPath = path.Trim();
query = query.Where(x => x.Path.Contains(normalizedPath));
}
query = query.Where(x =>
x.CreatedAt >= (from ?? DateTime.UtcNow.AddDays(-1)));
if (to.HasValue)
query = query.Where(x => x.CreatedAt <= to.Value);
var total = await query.CountAsync(cancellationToken);
var items = await query
.OrderByDescending(x => x.CreatedAt)
.Skip((page - 1) * pageSize)
.Take(pageSize)
.Select(x => new AuditLogItem(
x.Id,
x.UserName,
x.Method,
x.Path,
x.StatusCode,
x.IpAddress,
x.CreatedAt))
.ToListAsync(cancellationToken);
return Ok(new PagedResult<AuditLogItem>(items, total, page, pageSize));
}
[HttpGet("failed-jobs")]
public async Task<ActionResult<PagedResult<FailedBackgroundJobItem>>>
GetFailedJobs(
[FromQuery] int page = 1,
[FromQuery] int pageSize = 20,
[FromQuery] string? kind = null,
[FromQuery] DateTime? from = null,
[FromQuery] DateTime? to = null,
CancellationToken cancellationToken = default)
{
var pagingError = ValidatePaging(page, pageSize);
if (pagingError is not null) return pagingError;
var rangeError = ValidateRange(from, to);
if (rangeError is not null) return rangeError;
var normalizedKind = NormalizeJobKind(kind);
if (kind is not null && normalizedKind is null)
return ValidationProblem("后台任务类型无效。");
var effectiveFrom = from ?? DateTime.UtcNow.AddDays(-30);
var take = checked(page * pageSize);
var rows = new List<FailedBackgroundJobItem>();
var total = 0;
if (normalizedKind is null or "AutomaticSchedule")
{
var query = db.AutomaticScheduleJobs.AsNoTracking()
.Where(x =>
x.Status == AutomaticScheduleJobStatus.Failed &&
x.CreatedAt >= effectiveFrom);
if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value);
total += await query.CountAsync(cancellationToken);
rows.AddRange(await query
.OrderByDescending(x => x.CompletedAt ?? x.UpdatedAt)
.Take(take)
.Select(x => new FailedBackgroundJobItem(
x.Id,
"AutomaticSchedule",
"自动排课",
x.SchedulePlan == null ? "排课方案" : x.SchedulePlan.Name,
x.ErrorMessage ?? "任务失败但未记录错误详情。",
x.CreatedAt,
x.StartedAt,
x.CompletedAt,
0))
.ToListAsync(cancellationToken));
}
if (normalizedKind is null or "SchedulePublish")
{
var query = db.SchedulePublishJobs.AsNoTracking()
.Where(x =>
x.Status == SchedulePublishJobStatus.Failed &&
x.CreatedAt >= effectiveFrom);
if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value);
total += await query.CountAsync(cancellationToken);
rows.AddRange(await query
.OrderByDescending(x => x.CompletedAt ?? x.UpdatedAt)
.Take(take)
.Select(x => new FailedBackgroundJobItem(
x.Id,
"SchedulePublish",
"课表发布",
x.SchedulePlan == null ? "排课方案" : x.SchedulePlan.Name,
x.ErrorMessage ?? "任务失败但未记录错误详情。",
x.CreatedAt,
x.StartedAt,
x.CompletedAt,
0))
.ToListAsync(cancellationToken));
}
if (normalizedKind is null or "MakeupExamAuto")
{
var query = db.MakeupExamAutoJobs.AsNoTracking()
.Where(x =>
x.Status == MakeupExamAutoJobStatus.Failed &&
x.CreatedAt >= effectiveFrom);
if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value);
total += await query.CountAsync(cancellationToken);
rows.AddRange(await query
.OrderByDescending(x => x.CompletedAt ?? x.UpdatedAt)
.Take(take)
.Select(x => new FailedBackgroundJobItem(
x.Id,
"MakeupExamAuto",
"补考自动安排",
x.MakeupExamPlan == null ? "补考计划" : x.MakeupExamPlan.Name,
x.ErrorMessage ?? "任务失败但未记录错误详情。",
x.CreatedAt,
x.StartedAt,
x.CompletedAt,
0))
.ToListAsync(cancellationToken));
}
var pageItems = rows
.OrderByDescending(x => x.CompletedAt ?? x.CreatedAt)
.Skip((page - 1) * pageSize)
.Take(pageSize)
.ToArray();
if (pageItems.Length > 0)
{
var ids = pageItems.Select(x => x.Id).ToArray();
var attempts = await db.BackgroundJobOutboxMessages.AsNoTracking()
.Where(x => ids.Contains(x.JobId))
.Select(x => new { x.JobId, x.ProcessingAttempts })
.ToDictionaryAsync(x => x.JobId, x => x.ProcessingAttempts,
cancellationToken);
pageItems = pageItems
.Select(x => x with
{
ProcessingAttempts = attempts.GetValueOrDefault(x.Id)
})
.ToArray();
}
return Ok(new PagedResult<FailedBackgroundJobItem>(
pageItems,
total,
page,
pageSize));
}
[HttpGet("backups")]
public async Task<ActionResult<IReadOnlyCollection<BackupArtifact>>> GetBackups(
CancellationToken cancellationToken) =>
Ok(await backupService.ListAsync(cancellationToken));
[HttpPost("backups")]
public async Task<ActionResult<BackupArtifact>> CreateBackup(
CreateBackupRequest request,
CancellationToken cancellationToken)
{
try
{
var artifact = await backupService.CreateAsync(
request.Note,
cancellationToken);
return CreatedAtAction(
nameof(GetBackups),
new { id = artifact.Id },
artifact);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
return Problem(
title: "数据库备份失败",
detail: SafeMessage(exception),
statusCode: StatusCodes.Status503ServiceUnavailable);
}
}
[HttpPost("backups/{backupId}/restore-drill")]
public async Task<ActionResult<RestoreDrillResult>> RunRestoreDrill(
string backupId,
RestoreDrillRequest request,
CancellationToken cancellationToken)
{
if (!request.Confirmation.Equals(
"RESTORE_DRILL",
StringComparison.Ordinal))
{
return ValidationProblem(
"恢复演练必须明确确认,且不会覆盖当前业务数据库。");
}
try
{
return Ok(await backupService.RunRestoreDrillAsync(
backupId,
cancellationToken));
}
catch (FileNotFoundException)
{
return NotFound(new ProblemDetails
{
Title = "备份不存在",
Detail = "指定备份不存在或其文件已被移除。",
Status = StatusCodes.Status404NotFound
});
}
}
private async Task<IReadOnlyCollection<OperationalAlert>> BuildAlertsAsync(
OperationalHealthSnapshot health,
IReadOnlyCollection<BackupArtifact> backups,
int serverErrorCount,
int failedJobCount,
CancellationToken cancellationToken)
{
var alerts = health.Components
.Where(x => x.Status != "healthy")
.Select(x => new OperationalAlert(
$"health-{x.Key}",
x.Status == "unhealthy" ? "critical" : "warning",
"health",
$"{x.Label}状态异常",
x.Detail,
health.CheckedAt))
.ToList();
if (serverErrorCount > 0)
{
var latest = await db.AuditLogs.AsNoTracking()
.Where(x =>
x.CreatedAt >= DateTime.UtcNow.AddHours(-24) &&
x.StatusCode >= 500)
.OrderByDescending(x => x.CreatedAt)
.Select(x => new { x.Path, x.StatusCode, x.CreatedAt })
.FirstOrDefaultAsync(cancellationToken);
alerts.Add(new OperationalAlert(
"http-5xx",
"critical",
"audit",
$"过去 24 小时发生 {serverErrorCount} 次服务端错误",
latest is null
? "请检查服务日志定位异常。"
: $"最近一次为 {latest.StatusCode} {latest.Path}。",
latest?.CreatedAt ?? DateTime.UtcNow));
}
if (failedJobCount > 0)
{
alerts.Add(new OperationalAlert(
"failed-jobs",
"critical",
"jobs",
$"最近 7 天有 {failedJobCount} 个后台任务失败",
"任务已停止或达到重试上限,请在失败任务中查看错误详情。",
DateTime.UtcNow));
}
var retryingFailures = await db.BackgroundJobOutboxMessages.AsNoTracking()
.CountAsync(
x => x.State != BackgroundJobOutboxState.Completed &&
x.LastError != null,
cancellationToken);
if (retryingFailures > 0)
{
alerts.Add(new OperationalAlert(
"retrying-jobs",
"warning",
"jobs",
$"{retryingFailures} 个后台任务正在错误重试",
"任务队列仍会自动重试;若持续出现,请检查依赖服务与任务参数。",
DateTime.UtcNow));
}
var latestBackup = backups.FirstOrDefault();
if (latestBackup is null)
{
alerts.Add(new OperationalAlert(
"backup-missing",
"critical",
"backup",
"尚无可验证的数据库备份",
"立即创建首个备份,并在创建后执行一次隔离恢复演练。",
DateTime.UtcNow));
}
else
{
var ageHours = (DateTime.UtcNow - latestBackup.CreatedAt).TotalHours;
if (ageHours > options.BackupWarningHours)
{
alerts.Add(new OperationalAlert(
"backup-stale",
"warning",
"backup",
$"最近备份已超过 {options.BackupWarningHours} 小时",
$"最近备份创建于 {latestBackup.CreatedAt:u}。",
latestBackup.CreatedAt));
}
if (latestBackup.LastDrillSucceeded == false)
{
alerts.Add(new OperationalAlert(
"restore-drill-failed",
"critical",
"backup",
"最近一次恢复演练失败",
latestBackup.LastDrillDetail ?? "请重新运行演练并检查数据库工具日志。",
latestBackup.LastDrillAt ?? latestBackup.CreatedAt));
}
else if (!latestBackup.LastDrillAt.HasValue)
{
alerts.Add(new OperationalAlert(
"restore-drill-missing",
"warning",
"backup",
"最近备份尚未完成恢复演练",
"恢复演练只写入隔离数据库,不会覆盖当前业务数据。",
latestBackup.CreatedAt));
}
}
return alerts
.OrderBy(x => x.Severity == "critical" ? 0 : 1)
.ThenByDescending(x => x.OccurredAt)
.ToArray();
}
private async Task<int> CountFailedJobsAsync(
DateTime from,
CancellationToken cancellationToken) =>
await db.AutomaticScheduleJobs.AsNoTracking()
.CountAsync(
x => x.Status == AutomaticScheduleJobStatus.Failed &&
x.CreatedAt >= from,
cancellationToken) +
await db.SchedulePublishJobs.AsNoTracking()
.CountAsync(
x => x.Status == SchedulePublishJobStatus.Failed &&
x.CreatedAt >= from,
cancellationToken) +
await db.MakeupExamAutoJobs.AsNoTracking()
.CountAsync(
x => x.Status == MakeupExamAutoJobStatus.Failed &&
x.CreatedAt >= from,
cancellationToken);
private ActionResult? ValidatePaging(int page, int pageSize)
{
if (page is < 1 or > 100000 || pageSize is < 1 or > 100)
{
return ValidationProblem(
"页码必须在 1 到 100000 之间,每页数量必须在 1 到 100 之间。");
}
return null;
}
private ActionResult? ValidateRange(DateTime? from, DateTime? to)
{
if (from.HasValue && to.HasValue && from.Value > to.Value)
return ValidationProblem("开始时间不能晚于结束时间。");
return null;
}
private static string? NormalizeJobKind(string? kind)
{
if (string.IsNullOrWhiteSpace(kind)) return null;
return kind.Trim() switch
{
"AutomaticSchedule" => "AutomaticSchedule",
"SchedulePublish" => "SchedulePublish",
"MakeupExamAuto" => "MakeupExamAuto",
_ => null
};
}
private static string SafeMessage(Exception exception)
{
var message = exception.GetBaseException().Message;
return message.Length <= 500 ? message : message[..500];
}
}
public sealed record AuditLogItem(
Guid Id,
string? UserName,
string Method,
string Path,
int StatusCode,
string? IpAddress,
DateTime CreatedAt);
public sealed record FailedBackgroundJobItem(
Guid Id,
string Kind,
string KindLabel,
string Context,
string ErrorMessage,
DateTime CreatedAt,
DateTime? StartedAt,
DateTime? CompletedAt,
int ProcessingAttempts);
public sealed record OperationalAlert(
string Id,
string Severity,
string Source,
string Title,
string Detail,
DateTime OccurredAt);
public sealed record OperationsCounters(
int AuditEvents24Hours,
int ServerErrors24Hours,
int FailedJobs7Days,
int CriticalAlerts);
public sealed record OperationsSummary(
DateTime GeneratedAt,
OperationalHealthSnapshot Health,
OperationsCounters Counters,
IReadOnlyCollection<OperationalAlert> Alerts,
BackupArtifact? LatestBackup);
public sealed record CreateBackupRequest([MaxLength(200)] string? Note);
public sealed record RestoreDrillRequest([Required] string Confirmation);
@@ -0,0 +1,584 @@
using System.Diagnostics;
using System.Security.Cryptography;
using System.Text.Json;
using Jiaowu.Api.Infrastructure.Persistence;
using Microsoft.Data.Sqlite;
using MySql.Data.MySqlClient;
namespace Jiaowu.Api.Infrastructure.Operations;
public sealed record BackupArtifact(
string Id,
string FileName,
string Provider,
DateTime CreatedAt,
long SizeBytes,
string Sha256,
string? Note,
DateTime? LastDrillAt,
bool? LastDrillSucceeded,
string? LastDrillDetail,
long? LastDrillDurationMilliseconds);
public sealed record RestoreDrillResult(
string BackupId,
bool Succeeded,
DateTime CompletedAt,
string Detail,
long DurationMilliseconds,
int? TableCount);
public sealed class DatabaseBackupService(
DatabaseOptions databaseOptions,
OperationsOptions options,
IConfiguration configuration,
IHostEnvironment environment,
ILogger<DatabaseBackupService> logger)
{
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
{
WriteIndented = true
};
private readonly SemaphoreSlim operationLock = new(1, 1);
private readonly string backupDirectory = ResolveBackupDirectory(
options.BackupDirectory,
environment.ContentRootPath);
public async Task<IReadOnlyCollection<BackupArtifact>> ListAsync(
CancellationToken cancellationToken)
{
Directory.CreateDirectory(backupDirectory);
var items = new List<BackupArtifact>();
foreach (var metadataPath in Directory.EnumerateFiles(
backupDirectory,
"*.metadata.json",
SearchOption.TopDirectoryOnly))
{
cancellationToken.ThrowIfCancellationRequested();
try
{
await using var stream = File.OpenRead(metadataPath);
var artifact = await JsonSerializer.DeserializeAsync<BackupArtifact>(
stream,
JsonOptions,
cancellationToken);
if (artifact is not null &&
File.Exists(Path.Combine(backupDirectory, artifact.FileName)))
{
items.Add(artifact);
}
}
catch (Exception exception) when (
exception is IOException or UnauthorizedAccessException or JsonException)
{
logger.LogWarning(
exception,
"Unable to read backup metadata {MetadataFile}.",
Path.GetFileName(metadataPath));
}
}
return items
.OrderByDescending(x => x.CreatedAt)
.ToArray();
}
public async Task<BackupArtifact> CreateAsync(
string? note,
CancellationToken cancellationToken)
{
await operationLock.WaitAsync(cancellationToken);
try
{
Directory.CreateDirectory(backupDirectory);
var createdAt = DateTime.UtcNow;
var id = $"{createdAt:yyyyMMddHHmmss}-{Guid.NewGuid():N}"[..29];
var provider = NormalizeProvider(databaseOptions.Provider);
var extension = provider == "SQLite" ? ".sqlite" : ".sql";
var fileName = $"jiaowu-{id}{extension}";
var backupPath = Path.Combine(backupDirectory, fileName);
try
{
if (provider == "SQLite")
await CreateSqliteBackupAsync(backupPath, cancellationToken);
else
await CreateMySqlBackupAsync(backupPath, cancellationToken);
var fileInfo = new FileInfo(backupPath);
var artifact = new BackupArtifact(
id,
fileName,
provider,
createdAt,
fileInfo.Length,
await ComputeHashAsync(backupPath, cancellationToken),
NormalizeNote(note),
null,
null,
null,
null);
await WriteMetadataAsync(artifact, cancellationToken);
return artifact;
}
catch
{
if (File.Exists(backupPath))
File.Delete(backupPath);
throw;
}
}
finally
{
operationLock.Release();
}
}
public async Task<RestoreDrillResult> RunRestoreDrillAsync(
string backupId,
CancellationToken cancellationToken)
{
await operationLock.WaitAsync(cancellationToken);
try
{
var artifact = (await ListAsync(cancellationToken))
.SingleOrDefault(x => x.Id.Equals(backupId, StringComparison.Ordinal));
if (artifact is null)
throw new FileNotFoundException("未找到指定备份。");
var backupPath = Path.Combine(backupDirectory, artifact.FileName);
var actualHash = await ComputeHashAsync(backupPath, cancellationToken);
if (!CryptographicOperations.FixedTimeEquals(
Convert.FromHexString(artifact.Sha256),
Convert.FromHexString(actualHash)))
{
var damaged = await CompleteDrillAsync(
artifact,
false,
"备份文件校验和不一致,恢复演练已中止。",
0,
null,
cancellationToken);
return damaged;
}
var stopwatch = Stopwatch.StartNew();
RestoreDrillResult result;
try
{
var tableCount = artifact.Provider == "SQLite"
? await DrillSqliteAsync(backupPath, cancellationToken)
: await DrillMySqlAsync(backupPath, cancellationToken);
stopwatch.Stop();
result = await CompleteDrillAsync(
artifact,
true,
$"已在隔离数据库完成恢复并通过完整性检查,共发现 {tableCount} 张业务表。",
stopwatch.ElapsedMilliseconds,
tableCount,
cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
stopwatch.Stop();
logger.LogError(
exception,
"Restore drill failed for backup {BackupId}.",
artifact.Id);
result = await CompleteDrillAsync(
artifact,
false,
$"恢复演练失败:{SafeMessage(exception)}",
stopwatch.ElapsedMilliseconds,
null,
cancellationToken);
}
return result;
}
finally
{
operationLock.Release();
}
}
private async Task CreateSqliteBackupAsync(
string backupPath,
CancellationToken cancellationToken)
{
var sourceBuilder = new SqliteConnectionStringBuilder(
configuration.GetConnectionString("SQLite")
?? throw new InvalidOperationException("缺少 ConnectionStrings:SQLite。"));
if (!Path.IsPathRooted(sourceBuilder.DataSource))
{
sourceBuilder.DataSource = Path.GetFullPath(
sourceBuilder.DataSource,
environment.ContentRootPath);
}
var destinationBuilder = new SqliteConnectionStringBuilder
{
DataSource = backupPath,
Mode = SqliteOpenMode.ReadWriteCreate,
Pooling = false
};
await using var source = new SqliteConnection(sourceBuilder.ConnectionString);
await using var destination = new SqliteConnection(destinationBuilder.ConnectionString);
await source.OpenAsync(cancellationToken);
await destination.OpenAsync(cancellationToken);
source.BackupDatabase(destination);
}
private async Task CreateMySqlBackupAsync(
string backupPath,
CancellationToken cancellationToken)
{
var connection = GetMySqlConnectionBuilder();
var arguments = new List<string>
{
"--protocol=tcp",
$"--host={connection.Server}",
$"--port={connection.Port}",
$"--user={connection.UserID}",
"--single-transaction",
"--quick",
"--routines",
"--triggers",
"--events",
"--hex-blob",
"--default-character-set=utf8mb4"
};
arguments.AddRange(options.MySqlAdditionalArguments);
arguments.Add(connection.Database);
await using var output = new FileStream(
backupPath,
FileMode.CreateNew,
FileAccess.Write,
FileShare.None,
81920,
FileOptions.Asynchronous);
await RunToolAsync(
options.MySqlDumpPath,
arguments,
connection.Password,
standardInput: null,
standardOutput: output,
cancellationToken);
}
private static async Task<int> DrillSqliteAsync(
string backupPath,
CancellationToken cancellationToken)
{
var drillPath = Path.Combine(
Path.GetDirectoryName(backupPath)!,
$".restore-drill-{Guid.NewGuid():N}.sqlite");
try
{
var sourceBuilder = new SqliteConnectionStringBuilder
{
DataSource = backupPath,
Mode = SqliteOpenMode.ReadOnly,
Pooling = false
};
var drillBuilder = new SqliteConnectionStringBuilder
{
DataSource = drillPath,
Mode = SqliteOpenMode.ReadWriteCreate,
Pooling = false
};
await using var source = new SqliteConnection(sourceBuilder.ConnectionString);
await using var drill = new SqliteConnection(drillBuilder.ConnectionString);
await source.OpenAsync(cancellationToken);
await drill.OpenAsync(cancellationToken);
source.BackupDatabase(drill);
await using var integrity = drill.CreateCommand();
integrity.CommandText = "PRAGMA integrity_check;";
var integrityResult = Convert.ToString(
await integrity.ExecuteScalarAsync(cancellationToken));
if (!string.Equals(integrityResult, "ok", StringComparison.OrdinalIgnoreCase))
throw new InvalidDataException($"SQLite 完整性检查返回 {integrityResult ?? ""}。");
await using var tables = drill.CreateCommand();
tables.CommandText =
"SELECT COUNT(*) FROM sqlite_master " +
"WHERE type = 'table' AND name NOT LIKE 'sqlite_%';";
return Convert.ToInt32(await tables.ExecuteScalarAsync(cancellationToken));
}
finally
{
if (File.Exists(drillPath))
File.Delete(drillPath);
if (File.Exists($"{drillPath}-shm"))
File.Delete($"{drillPath}-shm");
if (File.Exists($"{drillPath}-wal"))
File.Delete($"{drillPath}-wal");
}
}
private async Task<int> DrillMySqlAsync(
string backupPath,
CancellationToken cancellationToken)
{
var connection = GetMySqlConnectionBuilder();
var drillDatabase = $"jiaowu_restore_drill_{DateTime.UtcNow:yyyyMMddHHmmss}_" +
Guid.NewGuid().ToString("N")[..8];
var adminBuilder = new MySqlConnectionStringBuilder(connection.ConnectionString)
{
Database = ""
};
await using var admin = new MySqlConnection(adminBuilder.ConnectionString);
await admin.OpenAsync(cancellationToken);
try
{
await using (var create = admin.CreateCommand())
{
create.CommandText =
$"CREATE DATABASE `{drillDatabase}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;";
await create.ExecuteNonQueryAsync(cancellationToken);
}
var arguments = new List<string>
{
"--protocol=tcp",
$"--host={connection.Server}",
$"--port={connection.Port}",
$"--user={connection.UserID}",
"--default-character-set=utf8mb4"
};
arguments.AddRange(options.MySqlAdditionalArguments);
arguments.Add($"--database={drillDatabase}");
await using (var input = new FileStream(
backupPath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
81920,
FileOptions.Asynchronous))
{
await RunToolAsync(
options.MySqlClientPath,
arguments,
connection.Password,
input,
standardOutput: null,
cancellationToken);
}
await using var count = admin.CreateCommand();
count.CommandText =
"SELECT COUNT(*) FROM information_schema.tables " +
"WHERE table_schema = @schema AND table_type = 'BASE TABLE';";
count.Parameters.AddWithValue("@schema", drillDatabase);
return Convert.ToInt32(await count.ExecuteScalarAsync(cancellationToken));
}
finally
{
try
{
await using var drop = admin.CreateCommand();
drop.CommandText = $"DROP DATABASE IF EXISTS `{drillDatabase}`;";
await drop.ExecuteNonQueryAsync(CancellationToken.None);
}
catch (Exception exception)
{
logger.LogCritical(
exception,
"Unable to remove isolated restore drill database {DatabaseName}.",
drillDatabase);
}
}
}
private async Task RunToolAsync(
string executable,
IReadOnlyCollection<string> arguments,
string password,
Stream? standardInput,
Stream? standardOutput,
CancellationToken cancellationToken)
{
var startInfo = new ProcessStartInfo
{
FileName = executable,
UseShellExecute = false,
RedirectStandardError = true,
RedirectStandardInput = standardInput is not null,
RedirectStandardOutput = standardOutput is not null,
CreateNoWindow = true
};
foreach (var argument in arguments)
startInfo.ArgumentList.Add(argument);
if (!string.IsNullOrEmpty(password))
startInfo.Environment["MYSQL_PWD"] = password;
using var process = Process.Start(startInfo)
?? throw new InvalidOperationException($"无法启动数据库工具 {executable}。");
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
timeout.CancelAfter(TimeSpan.FromMinutes(options.ToolTimeoutMinutes));
var errorTask = process.StandardError.ReadToEndAsync(timeout.Token);
var inputTask = standardInput is null
? Task.CompletedTask
: CopyInputAsync(standardInput, process.StandardInput.BaseStream, timeout.Token);
var outputTask = standardOutput is null
? Task.CompletedTask
: process.StandardOutput.BaseStream.CopyToAsync(
standardOutput,
timeout.Token);
try
{
await Task.WhenAll(
process.WaitForExitAsync(timeout.Token),
inputTask,
outputTask);
}
catch (OperationCanceledException)
{
if (!process.HasExited)
process.Kill(entireProcessTree: true);
throw;
}
var error = await errorTask;
if (process.ExitCode != 0)
throw new InvalidOperationException(
$"数据库工具执行失败(退出码 {process.ExitCode}):{TrimToolError(error)}");
}
private static async Task CopyInputAsync(
Stream input,
Stream processInput,
CancellationToken cancellationToken)
{
await input.CopyToAsync(processInput, cancellationToken);
await processInput.FlushAsync(cancellationToken);
processInput.Close();
}
private MySqlConnectionStringBuilder GetMySqlConnectionBuilder()
{
var value = configuration.GetConnectionString("OperationsMySql");
if (string.IsNullOrWhiteSpace(value))
{
throw new InvalidOperationException(
"MySQL 备份与恢复演练必须配置独立的 " +
"ConnectionStrings:OperationsMySql 运维账号,不能复用日常业务账号。");
}
var builder = new MySqlConnectionStringBuilder(value);
if (string.IsNullOrWhiteSpace(builder.Database))
throw new InvalidOperationException(
"OperationsMySql 连接字符串未指定业务数据库名称。");
return builder;
}
private async Task<RestoreDrillResult> CompleteDrillAsync(
BackupArtifact artifact,
bool succeeded,
string detail,
long durationMilliseconds,
int? tableCount,
CancellationToken cancellationToken)
{
var completedAt = DateTime.UtcNow;
var updated = artifact with
{
LastDrillAt = completedAt,
LastDrillSucceeded = succeeded,
LastDrillDetail = detail,
LastDrillDurationMilliseconds = durationMilliseconds
};
await WriteMetadataAsync(updated, cancellationToken);
return new RestoreDrillResult(
artifact.Id,
succeeded,
completedAt,
detail,
durationMilliseconds,
tableCount);
}
private async Task WriteMetadataAsync(
BackupArtifact artifact,
CancellationToken cancellationToken)
{
var metadataPath = Path.Combine(
backupDirectory,
$"{artifact.Id}.metadata.json");
var temporaryPath = $"{metadataPath}.{Guid.NewGuid():N}.tmp";
try
{
await using (var stream = new FileStream(
temporaryPath,
FileMode.CreateNew,
FileAccess.Write,
FileShare.None,
16384,
FileOptions.Asynchronous))
{
await JsonSerializer.SerializeAsync(
stream,
artifact,
JsonOptions,
cancellationToken);
}
File.Move(temporaryPath, metadataPath, overwrite: true);
}
finally
{
if (File.Exists(temporaryPath))
File.Delete(temporaryPath);
}
}
private static async Task<string> ComputeHashAsync(
string path,
CancellationToken cancellationToken)
{
await using var stream = File.OpenRead(path);
return Convert.ToHexString(
await SHA256.HashDataAsync(stream, cancellationToken));
}
private static string ResolveBackupDirectory(
string configuredPath,
string contentRoot)
{
if (string.IsNullOrWhiteSpace(configuredPath))
throw new InvalidOperationException("Operations:BackupDirectory 不能为空。");
return Path.IsPathRooted(configuredPath)
? Path.GetFullPath(configuredPath)
: Path.GetFullPath(configuredPath, contentRoot);
}
private static string NormalizeProvider(string provider) =>
provider.Equals("SQLite", StringComparison.OrdinalIgnoreCase)
? "SQLite"
: provider.Equals("MySql", StringComparison.OrdinalIgnoreCase)
? "MySql"
: throw new InvalidOperationException($"不支持数据库 Provider '{provider}'。");
private static string? NormalizeNote(string? note)
{
if (string.IsNullOrWhiteSpace(note)) return null;
var trimmed = note.Trim();
return trimmed.Length <= 200 ? trimmed : trimmed[..200];
}
private static string SafeMessage(Exception exception)
{
var message = exception.GetBaseException().Message;
return message.Length <= 500 ? message : message[..500];
}
private static string TrimToolError(string error)
{
var trimmed = error.Trim();
if (trimmed.Length == 0) return "未返回错误详情。";
return trimmed.Length <= 500 ? trimmed : trimmed[..500];
}
}
@@ -0,0 +1,200 @@
using System.Diagnostics;
using Jiaowu.Api.Infrastructure.BackgroundJobs;
using Jiaowu.Api.Infrastructure.Caching;
using Jiaowu.Api.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Caching.Distributed;
namespace Jiaowu.Api.Infrastructure.Operations;
public sealed record OperationalComponentHealth(
string Key,
string Label,
string Status,
string Backend,
long? LatencyMilliseconds,
string Detail);
public sealed record OperationalHealthSnapshot(
DateTime CheckedAt,
string OverallStatus,
IReadOnlyCollection<OperationalComponentHealth> Components,
BackgroundJobBacklogSnapshot? Backlog);
public sealed class OperationalHealthService(
AppDbContext db,
IServiceProvider services,
IBackgroundJobTransport transport,
BackgroundJobMonitoringService monitoring,
DatabaseOptions databaseOptions,
AppCacheOptions cacheOptions,
IConfiguration configuration)
{
public async Task<OperationalHealthSnapshot> CheckAsync(
CancellationToken cancellationToken)
{
var database = await CheckDatabaseAsync(cancellationToken);
var cache = await CheckCacheAsync(cancellationToken);
var (messaging, backlog) = await CheckMessagingAsync(cancellationToken);
var components = new[] { database, cache, messaging };
var overall = components.Any(x => x.Status == "unhealthy")
? "unhealthy"
: components.Any(x => x.Status == "warning")
? "warning"
: "healthy";
return new OperationalHealthSnapshot(
DateTime.UtcNow,
overall,
components,
backlog);
}
private async Task<OperationalComponentHealth> CheckDatabaseAsync(
CancellationToken cancellationToken)
{
var stopwatch = Stopwatch.StartNew();
try
{
var canConnect = await db.Database.CanConnectAsync(cancellationToken);
stopwatch.Stop();
return canConnect
? new OperationalComponentHealth(
"database",
"数据库",
"healthy",
databaseOptions.Provider,
stopwatch.ElapsedMilliseconds,
"连接与基础查询正常。")
: new OperationalComponentHealth(
"database",
"数据库",
"unhealthy",
databaseOptions.Provider,
stopwatch.ElapsedMilliseconds,
"无法建立数据库连接。");
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
stopwatch.Stop();
return new OperationalComponentHealth(
"database",
"数据库",
"unhealthy",
databaseOptions.Provider,
stopwatch.ElapsedMilliseconds,
SafeMessage(exception));
}
}
private async Task<OperationalComponentHealth> CheckCacheAsync(
CancellationToken cancellationToken)
{
if (!cacheOptions.Enabled)
{
return new OperationalComponentHealth(
"cache",
"缓存",
"warning",
"disabled",
null,
"缓存已通过配置关闭,所有查询将直接访问数据源。");
}
var hasRedisConfiguration = !string.IsNullOrWhiteSpace(
configuration.GetConnectionString("Redis"));
var distributedCache = services.GetService<IDistributedCache>();
if (!hasRedisConfiguration || distributedCache is null)
{
return new OperationalComponentHealth(
"cache",
"缓存",
"healthy",
"memory",
null,
"使用进程内混合缓存;服务重启后缓存会自然重建。");
}
var stopwatch = Stopwatch.StartNew();
try
{
await distributedCache.GetAsync(
"jiaowu:operations:health-probe",
cancellationToken);
stopwatch.Stop();
return new OperationalComponentHealth(
"cache",
"缓存",
"healthy",
"redis",
stopwatch.ElapsedMilliseconds,
"Redis 连接与读取探针正常。");
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
stopwatch.Stop();
return new OperationalComponentHealth(
"cache",
"缓存",
"unhealthy",
"redis",
stopwatch.ElapsedMilliseconds,
SafeMessage(exception));
}
}
private async Task<(OperationalComponentHealth Component,
BackgroundJobBacklogSnapshot? Backlog)> CheckMessagingAsync(
CancellationToken cancellationToken)
{
var stopwatch = Stopwatch.StartNew();
BackgroundJobBacklogSnapshot? backlog = null;
try
{
var healthy = await transport.CheckHealthAsync(cancellationToken);
backlog = await monitoring.GetSnapshotAsync(cancellationToken);
stopwatch.Stop();
var stuck = backlog.ExpiredLeases > 0 ||
backlog.OldestUnfinishedAgeSeconds > 1800;
var status = !healthy
? "unhealthy"
: stuck
? "warning"
: "healthy";
var detail = !healthy
? "后台任务传输不可用。"
: stuck
? $"发现 {backlog.ExpiredLeases} 个过期租约,最早未完成任务已等待 " +
$"{Math.Round(backlog.OldestUnfinishedAgeSeconds ?? 0)} 秒。"
: $"待发布 {backlog.Pending + backlog.Publishing}" +
$"待处理 {backlog.Published + backlog.Processing}。";
return (
new OperationalComponentHealth(
"messaging",
"后台任务通道",
status,
transport.IsDurable ? "rabbitmq" : "memory",
stopwatch.ElapsedMilliseconds,
detail),
backlog);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
stopwatch.Stop();
return (
new OperationalComponentHealth(
"messaging",
"后台任务通道",
"unhealthy",
transport.IsDurable ? "rabbitmq" : "memory",
stopwatch.ElapsedMilliseconds,
SafeMessage(exception)),
backlog);
}
}
private static string SafeMessage(Exception exception)
{
var message = exception.GetBaseException().Message;
return message.Length <= 300 ? message : message[..300];
}
}
@@ -0,0 +1,13 @@
namespace Jiaowu.Api.Infrastructure.Operations;
public sealed class OperationsOptions
{
public const string SectionName = "Operations";
public string BackupDirectory { get; set; } = "data/backups";
public int BackupWarningHours { get; set; } = 24;
public int ToolTimeoutMinutes { get; set; } = 30;
public string MySqlDumpPath { get; set; } = "mysqldump";
public string MySqlClientPath { get; set; } = "mysql";
public string[] MySqlAdditionalArguments { get; set; } = [];
}
+21
View File
@@ -8,6 +8,7 @@ using Jiaowu.Api.Infrastructure.Caching;
using Jiaowu.Api.Infrastructure.Exams;
using Jiaowu.Api.Infrastructure.Middleware;
using Jiaowu.Api.Infrastructure.OfficialDocuments;
using Jiaowu.Api.Infrastructure.Operations;
using Jiaowu.Api.Infrastructure.Persistence;
using Jiaowu.Api.Infrastructure.Scheduling;
using Jiaowu.Api.Infrastructure.Timetables;
@@ -74,6 +75,9 @@ var officialDocumentOptions = builder.Configuration
var backgroundJobOptions = builder.Configuration
.GetSection(BackgroundJobOptions.SectionName)
.Get<BackgroundJobOptions>() ?? new BackgroundJobOptions();
var operationsOptions = builder.Configuration
.GetSection(OperationsOptions.SectionName)
.Get<OperationsOptions>() ?? new OperationsOptions();
var rabbitMqOptions = builder.Configuration
.GetSection(RabbitMqOptions.SectionName)
.Get<RabbitMqOptions>() ?? new RabbitMqOptions();
@@ -167,10 +171,25 @@ if (backgroundJobOptions.UsesRabbitMq &&
"生产环境启用 RabbitMQ 时不能使用默认 guest 凭据。");
}
if (string.IsNullOrWhiteSpace(operationsOptions.BackupDirectory) ||
operationsOptions.BackupWarningHours is < 1 or > 8760 ||
operationsOptions.ToolTimeoutMinutes is < 1 or > 240 ||
string.IsNullOrWhiteSpace(operationsOptions.MySqlDumpPath) ||
string.IsNullOrWhiteSpace(operationsOptions.MySqlClientPath) ||
operationsOptions.MySqlAdditionalArguments.Length > 20 ||
operationsOptions.MySqlAdditionalArguments.Any(argument =>
string.IsNullOrWhiteSpace(argument) ||
argument.Length > 300 ||
!argument.StartsWith("--", StringComparison.Ordinal)))
{
throw new InvalidOperationException("Operations 运维与备份配置超出允许范围。");
}
builder.Services.AddSingleton(databaseOptions);
builder.Services.AddSingleton(cacheOptions);
builder.Services.AddSingleton(officialDocumentOptions);
builder.Services.AddSingleton(backgroundJobOptions);
builder.Services.AddSingleton(operationsOptions);
builder.Services.AddSingleton(rabbitMqOptions);
builder.Services.Configure<OfficialDocumentOptions>(
builder.Configuration.GetSection(OfficialDocumentOptions.SectionName));
@@ -277,6 +296,8 @@ builder.Services.AddScoped<MakeupExamArrangementService>();
builder.Services.AddScoped<MakeupExamAutoJobProcessor>();
builder.Services.AddSingleton<BackgroundJobTelemetry>();
builder.Services.AddScoped<BackgroundJobMonitoringService>();
builder.Services.AddScoped<OperationalHealthService>();
builder.Services.AddSingleton<DatabaseBackupService>();
builder.Services.AddSingleton<BackgroundJobRunner>();
if (backgroundJobOptions.UsesRabbitMq)
{
+8
View File
@@ -19,6 +19,14 @@
"AnalyticsLocalExpirationSeconds": 30,
"MaximumPayloadKilobytes": 2048
},
"Operations": {
"BackupDirectory": "data/backups",
"BackupWarningHours": 24,
"ToolTimeoutMinutes": 30,
"MySqlDumpPath": "mysqldump",
"MySqlClientPath": "mysql",
"MySqlAdditionalArguments": []
},
"BackgroundJobs": {
"Transport": "InMemory",
"PollIntervalMilliseconds": 500,