“运维与审计控制台”。
主要能力： SuperAdmin 专用入口：组织与权限 → 运维与审计。 操作日志分页查询，支持时间、账号、路径、方法和状态码筛选。 汇总自动排课、课表发布、补考安排三类失败后台任务。 实时检查数据库、缓存、任务通道及积压状态。 聚合 5xx、失败/重试任务、健康探针和备份时效告警。 SQLite 在线备份；MySQL 调用原生客户端备份。 SHA-256 校验及隔离数据库恢复演练，不覆盖业务库。 MySQL 强制使用独立运维连接，容器增加持久化备份卷与数据库客户端。
This commit is contained in:
@@ -0,0 +1,519 @@
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
using Jiaowu.Api.Contracts;
|
||||
using Jiaowu.Api.Domain.Academic;
|
||||
using Jiaowu.Api.Domain.Identity;
|
||||
using Jiaowu.Api.Domain.System;
|
||||
using Jiaowu.Api.Infrastructure.Operations;
|
||||
using Jiaowu.Api.Infrastructure.Persistence;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace Jiaowu.Api.Controllers;
|
||||
|
||||
[ApiController]
|
||||
[Authorize(Roles = SystemRoles.SuperAdmin)]
|
||||
[Route("api/operations")]
|
||||
public sealed class OperationsController(
|
||||
AppDbContext db,
|
||||
OperationalHealthService healthService,
|
||||
DatabaseBackupService backupService,
|
||||
OperationsOptions options) : ControllerBase
|
||||
{
|
||||
[HttpGet("summary")]
|
||||
public async Task<ActionResult<OperationsSummary>> GetSummary(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var since = DateTime.UtcNow.AddHours(-24);
|
||||
var health = await healthService.CheckAsync(cancellationToken);
|
||||
var backups = await backupService.ListAsync(cancellationToken);
|
||||
var auditCount = await db.AuditLogs.AsNoTracking()
|
||||
.CountAsync(x => x.CreatedAt >= since, cancellationToken);
|
||||
var serverErrorCount = await db.AuditLogs.AsNoTracking()
|
||||
.CountAsync(
|
||||
x => x.CreatedAt >= since && x.StatusCode >= 500,
|
||||
cancellationToken);
|
||||
var failedJobCount = await CountFailedJobsAsync(
|
||||
DateTime.UtcNow.AddDays(-7),
|
||||
cancellationToken);
|
||||
var alerts = await BuildAlertsAsync(
|
||||
health,
|
||||
backups,
|
||||
serverErrorCount,
|
||||
failedJobCount,
|
||||
cancellationToken);
|
||||
return Ok(new OperationsSummary(
|
||||
DateTime.UtcNow,
|
||||
health,
|
||||
new OperationsCounters(
|
||||
auditCount,
|
||||
serverErrorCount,
|
||||
failedJobCount,
|
||||
alerts.Count(x => x.Severity == "critical")),
|
||||
alerts,
|
||||
backups.FirstOrDefault()));
|
||||
}
|
||||
|
||||
[HttpGet("health")]
|
||||
public async Task<ActionResult<OperationalHealthSnapshot>> GetHealth(
|
||||
CancellationToken cancellationToken) =>
|
||||
Ok(await healthService.CheckAsync(cancellationToken));
|
||||
|
||||
[HttpGet("audit-logs")]
|
||||
public async Task<ActionResult<PagedResult<AuditLogItem>>> GetAuditLogs(
|
||||
[FromQuery] int page = 1,
|
||||
[FromQuery] int pageSize = 20,
|
||||
[FromQuery] string? method = null,
|
||||
[FromQuery] int? statusCode = null,
|
||||
[FromQuery] string? userName = null,
|
||||
[FromQuery] string? path = null,
|
||||
[FromQuery] DateTime? from = null,
|
||||
[FromQuery] DateTime? to = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var pagingError = ValidatePaging(page, pageSize);
|
||||
if (pagingError is not null) return pagingError;
|
||||
var rangeError = ValidateRange(from, to);
|
||||
if (rangeError is not null) return rangeError;
|
||||
|
||||
var query = db.AuditLogs.AsNoTracking().AsQueryable();
|
||||
if (!string.IsNullOrWhiteSpace(method))
|
||||
{
|
||||
var normalizedMethod = method.Trim().ToUpperInvariant();
|
||||
query = query.Where(x => x.Method == normalizedMethod);
|
||||
}
|
||||
if (statusCode.HasValue)
|
||||
query = query.Where(x => x.StatusCode == statusCode.Value);
|
||||
if (!string.IsNullOrWhiteSpace(userName))
|
||||
{
|
||||
var normalizedUser = userName.Trim();
|
||||
query = query.Where(x =>
|
||||
x.UserName != null && x.UserName.Contains(normalizedUser));
|
||||
}
|
||||
if (!string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
var normalizedPath = path.Trim();
|
||||
query = query.Where(x => x.Path.Contains(normalizedPath));
|
||||
}
|
||||
query = query.Where(x =>
|
||||
x.CreatedAt >= (from ?? DateTime.UtcNow.AddDays(-1)));
|
||||
if (to.HasValue)
|
||||
query = query.Where(x => x.CreatedAt <= to.Value);
|
||||
|
||||
var total = await query.CountAsync(cancellationToken);
|
||||
var items = await query
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.Skip((page - 1) * pageSize)
|
||||
.Take(pageSize)
|
||||
.Select(x => new AuditLogItem(
|
||||
x.Id,
|
||||
x.UserName,
|
||||
x.Method,
|
||||
x.Path,
|
||||
x.StatusCode,
|
||||
x.IpAddress,
|
||||
x.CreatedAt))
|
||||
.ToListAsync(cancellationToken);
|
||||
return Ok(new PagedResult<AuditLogItem>(items, total, page, pageSize));
|
||||
}
|
||||
|
||||
[HttpGet("failed-jobs")]
|
||||
public async Task<ActionResult<PagedResult<FailedBackgroundJobItem>>>
|
||||
GetFailedJobs(
|
||||
[FromQuery] int page = 1,
|
||||
[FromQuery] int pageSize = 20,
|
||||
[FromQuery] string? kind = null,
|
||||
[FromQuery] DateTime? from = null,
|
||||
[FromQuery] DateTime? to = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var pagingError = ValidatePaging(page, pageSize);
|
||||
if (pagingError is not null) return pagingError;
|
||||
var rangeError = ValidateRange(from, to);
|
||||
if (rangeError is not null) return rangeError;
|
||||
|
||||
var normalizedKind = NormalizeJobKind(kind);
|
||||
if (kind is not null && normalizedKind is null)
|
||||
return ValidationProblem("后台任务类型无效。");
|
||||
|
||||
var effectiveFrom = from ?? DateTime.UtcNow.AddDays(-30);
|
||||
var take = checked(page * pageSize);
|
||||
var rows = new List<FailedBackgroundJobItem>();
|
||||
var total = 0;
|
||||
|
||||
if (normalizedKind is null or "AutomaticSchedule")
|
||||
{
|
||||
var query = db.AutomaticScheduleJobs.AsNoTracking()
|
||||
.Where(x =>
|
||||
x.Status == AutomaticScheduleJobStatus.Failed &&
|
||||
x.CreatedAt >= effectiveFrom);
|
||||
if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value);
|
||||
total += await query.CountAsync(cancellationToken);
|
||||
rows.AddRange(await query
|
||||
.OrderByDescending(x => x.CompletedAt ?? x.UpdatedAt)
|
||||
.Take(take)
|
||||
.Select(x => new FailedBackgroundJobItem(
|
||||
x.Id,
|
||||
"AutomaticSchedule",
|
||||
"自动排课",
|
||||
x.SchedulePlan == null ? "排课方案" : x.SchedulePlan.Name,
|
||||
x.ErrorMessage ?? "任务失败但未记录错误详情。",
|
||||
x.CreatedAt,
|
||||
x.StartedAt,
|
||||
x.CompletedAt,
|
||||
0))
|
||||
.ToListAsync(cancellationToken));
|
||||
}
|
||||
|
||||
if (normalizedKind is null or "SchedulePublish")
|
||||
{
|
||||
var query = db.SchedulePublishJobs.AsNoTracking()
|
||||
.Where(x =>
|
||||
x.Status == SchedulePublishJobStatus.Failed &&
|
||||
x.CreatedAt >= effectiveFrom);
|
||||
if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value);
|
||||
total += await query.CountAsync(cancellationToken);
|
||||
rows.AddRange(await query
|
||||
.OrderByDescending(x => x.CompletedAt ?? x.UpdatedAt)
|
||||
.Take(take)
|
||||
.Select(x => new FailedBackgroundJobItem(
|
||||
x.Id,
|
||||
"SchedulePublish",
|
||||
"课表发布",
|
||||
x.SchedulePlan == null ? "排课方案" : x.SchedulePlan.Name,
|
||||
x.ErrorMessage ?? "任务失败但未记录错误详情。",
|
||||
x.CreatedAt,
|
||||
x.StartedAt,
|
||||
x.CompletedAt,
|
||||
0))
|
||||
.ToListAsync(cancellationToken));
|
||||
}
|
||||
|
||||
if (normalizedKind is null or "MakeupExamAuto")
|
||||
{
|
||||
var query = db.MakeupExamAutoJobs.AsNoTracking()
|
||||
.Where(x =>
|
||||
x.Status == MakeupExamAutoJobStatus.Failed &&
|
||||
x.CreatedAt >= effectiveFrom);
|
||||
if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value);
|
||||
total += await query.CountAsync(cancellationToken);
|
||||
rows.AddRange(await query
|
||||
.OrderByDescending(x => x.CompletedAt ?? x.UpdatedAt)
|
||||
.Take(take)
|
||||
.Select(x => new FailedBackgroundJobItem(
|
||||
x.Id,
|
||||
"MakeupExamAuto",
|
||||
"补考自动安排",
|
||||
x.MakeupExamPlan == null ? "补考计划" : x.MakeupExamPlan.Name,
|
||||
x.ErrorMessage ?? "任务失败但未记录错误详情。",
|
||||
x.CreatedAt,
|
||||
x.StartedAt,
|
||||
x.CompletedAt,
|
||||
0))
|
||||
.ToListAsync(cancellationToken));
|
||||
}
|
||||
|
||||
var pageItems = rows
|
||||
.OrderByDescending(x => x.CompletedAt ?? x.CreatedAt)
|
||||
.Skip((page - 1) * pageSize)
|
||||
.Take(pageSize)
|
||||
.ToArray();
|
||||
if (pageItems.Length > 0)
|
||||
{
|
||||
var ids = pageItems.Select(x => x.Id).ToArray();
|
||||
var attempts = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||
.Where(x => ids.Contains(x.JobId))
|
||||
.Select(x => new { x.JobId, x.ProcessingAttempts })
|
||||
.ToDictionaryAsync(x => x.JobId, x => x.ProcessingAttempts,
|
||||
cancellationToken);
|
||||
pageItems = pageItems
|
||||
.Select(x => x with
|
||||
{
|
||||
ProcessingAttempts = attempts.GetValueOrDefault(x.Id)
|
||||
})
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
return Ok(new PagedResult<FailedBackgroundJobItem>(
|
||||
pageItems,
|
||||
total,
|
||||
page,
|
||||
pageSize));
|
||||
}
|
||||
|
||||
[HttpGet("backups")]
|
||||
public async Task<ActionResult<IReadOnlyCollection<BackupArtifact>>> GetBackups(
|
||||
CancellationToken cancellationToken) =>
|
||||
Ok(await backupService.ListAsync(cancellationToken));
|
||||
|
||||
[HttpPost("backups")]
|
||||
public async Task<ActionResult<BackupArtifact>> CreateBackup(
|
||||
CreateBackupRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var artifact = await backupService.CreateAsync(
|
||||
request.Note,
|
||||
cancellationToken);
|
||||
return CreatedAtAction(
|
||||
nameof(GetBackups),
|
||||
new { id = artifact.Id },
|
||||
artifact);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
return Problem(
|
||||
title: "数据库备份失败",
|
||||
detail: SafeMessage(exception),
|
||||
statusCode: StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
}
|
||||
|
||||
[HttpPost("backups/{backupId}/restore-drill")]
|
||||
public async Task<ActionResult<RestoreDrillResult>> RunRestoreDrill(
|
||||
string backupId,
|
||||
RestoreDrillRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!request.Confirmation.Equals(
|
||||
"RESTORE_DRILL",
|
||||
StringComparison.Ordinal))
|
||||
{
|
||||
return ValidationProblem(
|
||||
"恢复演练必须明确确认，且不会覆盖当前业务数据库。");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return Ok(await backupService.RunRestoreDrillAsync(
|
||||
backupId,
|
||||
cancellationToken));
|
||||
}
|
||||
catch (FileNotFoundException)
|
||||
{
|
||||
return NotFound(new ProblemDetails
|
||||
{
|
||||
Title = "备份不存在",
|
||||
Detail = "指定备份不存在或其文件已被移除。",
|
||||
Status = StatusCodes.Status404NotFound
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyCollection<OperationalAlert>> BuildAlertsAsync(
|
||||
OperationalHealthSnapshot health,
|
||||
IReadOnlyCollection<BackupArtifact> backups,
|
||||
int serverErrorCount,
|
||||
int failedJobCount,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var alerts = health.Components
|
||||
.Where(x => x.Status != "healthy")
|
||||
.Select(x => new OperationalAlert(
|
||||
$"health-{x.Key}",
|
||||
x.Status == "unhealthy" ? "critical" : "warning",
|
||||
"health",
|
||||
$"{x.Label}状态异常",
|
||||
x.Detail,
|
||||
health.CheckedAt))
|
||||
.ToList();
|
||||
|
||||
if (serverErrorCount > 0)
|
||||
{
|
||||
var latest = await db.AuditLogs.AsNoTracking()
|
||||
.Where(x =>
|
||||
x.CreatedAt >= DateTime.UtcNow.AddHours(-24) &&
|
||||
x.StatusCode >= 500)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.Select(x => new { x.Path, x.StatusCode, x.CreatedAt })
|
||||
.FirstOrDefaultAsync(cancellationToken);
|
||||
alerts.Add(new OperationalAlert(
|
||||
"http-5xx",
|
||||
"critical",
|
||||
"audit",
|
||||
$"过去 24 小时发生 {serverErrorCount} 次服务端错误",
|
||||
latest is null
|
||||
? "请检查服务日志定位异常。"
|
||||
: $"最近一次为 {latest.StatusCode} {latest.Path}。",
|
||||
latest?.CreatedAt ?? DateTime.UtcNow));
|
||||
}
|
||||
|
||||
if (failedJobCount > 0)
|
||||
{
|
||||
alerts.Add(new OperationalAlert(
|
||||
"failed-jobs",
|
||||
"critical",
|
||||
"jobs",
|
||||
$"最近 7 天有 {failedJobCount} 个后台任务失败",
|
||||
"任务已停止或达到重试上限，请在失败任务中查看错误详情。",
|
||||
DateTime.UtcNow));
|
||||
}
|
||||
|
||||
var retryingFailures = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||
.CountAsync(
|
||||
x => x.State != BackgroundJobOutboxState.Completed &&
|
||||
x.LastError != null,
|
||||
cancellationToken);
|
||||
if (retryingFailures > 0)
|
||||
{
|
||||
alerts.Add(new OperationalAlert(
|
||||
"retrying-jobs",
|
||||
"warning",
|
||||
"jobs",
|
||||
$"{retryingFailures} 个后台任务正在错误重试",
|
||||
"任务队列仍会自动重试；若持续出现，请检查依赖服务与任务参数。",
|
||||
DateTime.UtcNow));
|
||||
}
|
||||
|
||||
var latestBackup = backups.FirstOrDefault();
|
||||
if (latestBackup is null)
|
||||
{
|
||||
alerts.Add(new OperationalAlert(
|
||||
"backup-missing",
|
||||
"critical",
|
||||
"backup",
|
||||
"尚无可验证的数据库备份",
|
||||
"立即创建首个备份，并在创建后执行一次隔离恢复演练。",
|
||||
DateTime.UtcNow));
|
||||
}
|
||||
else
|
||||
{
|
||||
var ageHours = (DateTime.UtcNow - latestBackup.CreatedAt).TotalHours;
|
||||
if (ageHours > options.BackupWarningHours)
|
||||
{
|
||||
alerts.Add(new OperationalAlert(
|
||||
"backup-stale",
|
||||
"warning",
|
||||
"backup",
|
||||
$"最近备份已超过 {options.BackupWarningHours} 小时",
|
||||
$"最近备份创建于 {latestBackup.CreatedAt:u}。",
|
||||
latestBackup.CreatedAt));
|
||||
}
|
||||
if (latestBackup.LastDrillSucceeded == false)
|
||||
{
|
||||
alerts.Add(new OperationalAlert(
|
||||
"restore-drill-failed",
|
||||
"critical",
|
||||
"backup",
|
||||
"最近一次恢复演练失败",
|
||||
latestBackup.LastDrillDetail ?? "请重新运行演练并检查数据库工具日志。",
|
||||
latestBackup.LastDrillAt ?? latestBackup.CreatedAt));
|
||||
}
|
||||
else if (!latestBackup.LastDrillAt.HasValue)
|
||||
{
|
||||
alerts.Add(new OperationalAlert(
|
||||
"restore-drill-missing",
|
||||
"warning",
|
||||
"backup",
|
||||
"最近备份尚未完成恢复演练",
|
||||
"恢复演练只写入隔离数据库，不会覆盖当前业务数据。",
|
||||
latestBackup.CreatedAt));
|
||||
}
|
||||
}
|
||||
|
||||
return alerts
|
||||
.OrderBy(x => x.Severity == "critical" ? 0 : 1)
|
||||
.ThenByDescending(x => x.OccurredAt)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private async Task<int> CountFailedJobsAsync(
|
||||
DateTime from,
|
||||
CancellationToken cancellationToken) =>
|
||||
await db.AutomaticScheduleJobs.AsNoTracking()
|
||||
.CountAsync(
|
||||
x => x.Status == AutomaticScheduleJobStatus.Failed &&
|
||||
x.CreatedAt >= from,
|
||||
cancellationToken) +
|
||||
await db.SchedulePublishJobs.AsNoTracking()
|
||||
.CountAsync(
|
||||
x => x.Status == SchedulePublishJobStatus.Failed &&
|
||||
x.CreatedAt >= from,
|
||||
cancellationToken) +
|
||||
await db.MakeupExamAutoJobs.AsNoTracking()
|
||||
.CountAsync(
|
||||
x => x.Status == MakeupExamAutoJobStatus.Failed &&
|
||||
x.CreatedAt >= from,
|
||||
cancellationToken);
|
||||
|
||||
private ActionResult? ValidatePaging(int page, int pageSize)
|
||||
{
|
||||
if (page is < 1 or > 100000 || pageSize is < 1 or > 100)
|
||||
{
|
||||
return ValidationProblem(
|
||||
"页码必须在 1 到 100000 之间，每页数量必须在 1 到 100 之间。");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private ActionResult? ValidateRange(DateTime? from, DateTime? to)
|
||||
{
|
||||
if (from.HasValue && to.HasValue && from.Value > to.Value)
|
||||
return ValidationProblem("开始时间不能晚于结束时间。");
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string? NormalizeJobKind(string? kind)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(kind)) return null;
|
||||
return kind.Trim() switch
|
||||
{
|
||||
"AutomaticSchedule" => "AutomaticSchedule",
|
||||
"SchedulePublish" => "SchedulePublish",
|
||||
"MakeupExamAuto" => "MakeupExamAuto",
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
|
||||
private static string SafeMessage(Exception exception)
|
||||
{
|
||||
var message = exception.GetBaseException().Message;
|
||||
return message.Length <= 500 ? message : message[..500];
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record AuditLogItem(
|
||||
Guid Id,
|
||||
string? UserName,
|
||||
string Method,
|
||||
string Path,
|
||||
int StatusCode,
|
||||
string? IpAddress,
|
||||
DateTime CreatedAt);
|
||||
|
||||
public sealed record FailedBackgroundJobItem(
|
||||
Guid Id,
|
||||
string Kind,
|
||||
string KindLabel,
|
||||
string Context,
|
||||
string ErrorMessage,
|
||||
DateTime CreatedAt,
|
||||
DateTime? StartedAt,
|
||||
DateTime? CompletedAt,
|
||||
int ProcessingAttempts);
|
||||
|
||||
public sealed record OperationalAlert(
|
||||
string Id,
|
||||
string Severity,
|
||||
string Source,
|
||||
string Title,
|
||||
string Detail,
|
||||
DateTime OccurredAt);
|
||||
|
||||
public sealed record OperationsCounters(
|
||||
int AuditEvents24Hours,
|
||||
int ServerErrors24Hours,
|
||||
int FailedJobs7Days,
|
||||
int CriticalAlerts);
|
||||
|
||||
public sealed record OperationsSummary(
|
||||
DateTime GeneratedAt,
|
||||
OperationalHealthSnapshot Health,
|
||||
OperationsCounters Counters,
|
||||
IReadOnlyCollection<OperationalAlert> Alerts,
|
||||
BackupArtifact? LatestBackup);
|
||||
|
||||
public sealed record CreateBackupRequest([MaxLength(200)] string? Note);
|
||||
|
||||
public sealed record RestoreDrillRequest([Required] string Confirmation);
|
||||
Reference in New Issue
Block a user