using System.ComponentModel.DataAnnotations; using Jiaowu.Api.Contracts; using Jiaowu.Api.Domain.Academic; using Jiaowu.Api.Domain.Identity; using Jiaowu.Api.Domain.System; using Jiaowu.Api.Infrastructure.Operations; using Jiaowu.Api.Infrastructure.Observability; using Jiaowu.Api.Infrastructure.Persistence; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; namespace Jiaowu.Api.Controllers; [ApiController] [Authorize(Roles = SystemRoles.SuperAdmin)] [Route("api/operations")] public sealed class OperationsController( AppDbContext db, OperationalHealthService healthService, DatabaseBackupService backupService, PerformanceReportService performanceReportService, OperationsOptions options) : ControllerBase { [HttpGet("performance")] public async Task> GetPerformance( [FromQuery] string? range = "1h", CancellationToken cancellationToken = default) { try { return Ok(await performanceReportService.GetAsync( range, cancellationToken)); } catch (ArgumentOutOfRangeException) { return ValidationProblem( "性能报表范围仅支持 15m、1h、24h 或 7d。"); } } [HttpGet("summary")] public async Task> GetSummary( CancellationToken cancellationToken) { var since = DateTime.UtcNow.AddHours(-24); var health = await healthService.CheckAsync(cancellationToken); var backups = await backupService.ListAsync(cancellationToken); var auditCount = await db.AuditLogs.AsNoTracking() .CountAsync(x => x.CreatedAt >= since, cancellationToken); var serverErrorCount = await db.AuditLogs.AsNoTracking() .CountAsync( x => x.CreatedAt >= since && x.StatusCode >= 500, cancellationToken); var failedJobCount = await CountFailedJobsAsync( DateTime.UtcNow.AddDays(-7), cancellationToken); var alerts = await BuildAlertsAsync( health, backups, serverErrorCount, failedJobCount, cancellationToken); return Ok(new OperationsSummary( DateTime.UtcNow, health, new OperationsCounters( auditCount, serverErrorCount, failedJobCount, alerts.Count(x => x.Severity == "critical")), alerts, backups.FirstOrDefault())); } [HttpGet("health")] public async Task> GetHealth( CancellationToken cancellationToken) => Ok(await healthService.CheckAsync(cancellationToken)); [HttpGet("audit-logs")] public async Task>> GetAuditLogs( [FromQuery] int page = 1, [FromQuery] int pageSize = 20, [FromQuery] string? method = null, [FromQuery] int? statusCode = null, [FromQuery] string? userName = null, [FromQuery] string? path = null, [FromQuery] DateTime? from = null, [FromQuery] DateTime? to = null, CancellationToken cancellationToken = default) { var pagingError = ValidatePaging(page, pageSize); if (pagingError is not null) return pagingError; var rangeError = ValidateRange(from, to); if (rangeError is not null) return rangeError; var query = db.AuditLogs.AsNoTracking().AsQueryable(); if (!string.IsNullOrWhiteSpace(method)) { var normalizedMethod = method.Trim().ToUpperInvariant(); query = query.Where(x => x.Method == normalizedMethod); } if (statusCode.HasValue) query = query.Where(x => x.StatusCode == statusCode.Value); if (!string.IsNullOrWhiteSpace(userName)) { var normalizedUser = userName.Trim(); query = query.Where(x => x.UserName != null && x.UserName.Contains(normalizedUser)); } if (!string.IsNullOrWhiteSpace(path)) { var normalizedPath = path.Trim(); query = query.Where(x => x.Path.Contains(normalizedPath)); } query = query.Where(x => x.CreatedAt >= (from ?? DateTime.UtcNow.AddDays(-1))); if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value); var total = await query.CountAsync(cancellationToken); var items = await query .OrderByDescending(x => x.CreatedAt) .Skip((page - 1) * pageSize) .Take(pageSize) .Select(x => new AuditLogItem( x.Id, x.UserName, x.Method, x.Path, x.StatusCode, x.IpAddress, x.CreatedAt)) .ToListAsync(cancellationToken); return Ok(new PagedResult(items, total, page, pageSize)); } [HttpGet("failed-jobs")] public async Task>> GetFailedJobs( [FromQuery] int page = 1, [FromQuery] int pageSize = 20, [FromQuery] string? kind = null, [FromQuery] DateTime? from = null, [FromQuery] DateTime? to = null, CancellationToken cancellationToken = default) { var pagingError = ValidatePaging(page, pageSize); if (pagingError is not null) return pagingError; var rangeError = ValidateRange(from, to); if (rangeError is not null) return rangeError; var normalizedKind = NormalizeJobKind(kind); if (kind is not null && normalizedKind is null) return ValidationProblem("后台任务类型无效。"); var effectiveFrom = from ?? DateTime.UtcNow.AddDays(-30); var take = checked(page * pageSize); var rows = new List(); var total = 0; if (normalizedKind is null or "AutomaticSchedule") { var query = db.AutomaticScheduleJobs.AsNoTracking() .Where(x => x.Status == AutomaticScheduleJobStatus.Failed && x.CreatedAt >= effectiveFrom); if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value); total += await query.CountAsync(cancellationToken); rows.AddRange(await query .OrderByDescending(x => x.CompletedAt ?? x.UpdatedAt) .Take(take) .Select(x => new FailedBackgroundJobItem( x.Id, "AutomaticSchedule", "自动排课", x.SchedulePlan == null ? "排课方案" : x.SchedulePlan.Name, x.ErrorMessage ?? "任务失败但未记录错误详情。", x.CreatedAt, x.StartedAt, x.CompletedAt, 0)) .ToListAsync(cancellationToken)); } if (normalizedKind is null or "SchedulePublish") { var query = db.SchedulePublishJobs.AsNoTracking() .Where(x => x.Status == SchedulePublishJobStatus.Failed && x.CreatedAt >= effectiveFrom); if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value); total += await query.CountAsync(cancellationToken); rows.AddRange(await query .OrderByDescending(x => x.CompletedAt ?? x.UpdatedAt) .Take(take) .Select(x => new FailedBackgroundJobItem( x.Id, "SchedulePublish", "课表发布", x.SchedulePlan == null ? "排课方案" : x.SchedulePlan.Name, x.ErrorMessage ?? "任务失败但未记录错误详情。", x.CreatedAt, x.StartedAt, x.CompletedAt, 0)) .ToListAsync(cancellationToken)); } if (normalizedKind is null or "MakeupExamAuto") { var query = db.MakeupExamAutoJobs.AsNoTracking() .Where(x => x.Status == MakeupExamAutoJobStatus.Failed && x.CreatedAt >= effectiveFrom); if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value); total += await query.CountAsync(cancellationToken); rows.AddRange(await query .OrderByDescending(x => x.CompletedAt ?? x.UpdatedAt) .Take(take) .Select(x => new FailedBackgroundJobItem( x.Id, "MakeupExamAuto", "补考自动安排", x.MakeupExamPlan == null ? "补考计划" : x.MakeupExamPlan.Name, x.ErrorMessage ?? "任务失败但未记录错误详情。", x.CreatedAt, x.StartedAt, x.CompletedAt, 0)) .ToListAsync(cancellationToken)); } if (normalizedKind is null or "ExamArrangement") { var query = db.ExamArrangementJobs.AsNoTracking() .Where(x => x.Status == ExamArrangementJobStatus.Failed && x.CreatedAt >= effectiveFrom); if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value); total += await query.CountAsync(cancellationToken); rows.AddRange(await query .OrderByDescending(x => x.CompletedAt ?? x.UpdatedAt) .Take(take) .Select(x => new FailedBackgroundJobItem( x.Id, "ExamArrangement", x.Kind == ExamArrangementKind.FormalExam ? "正式考试编排" : "补考编排", x.Kind == ExamArrangementKind.FormalExam ? "正式考试计划" : "补考计划", x.ErrorMessage ?? "任务失败但未记录错误详情。", x.CreatedAt, x.StartedAt, x.CompletedAt, 0)) .ToListAsync(cancellationToken)); } if (normalizedKind is null or "ExamPublish") { var query = db.ExamPublishJobs.AsNoTracking() .Where(x => x.Status == ExamPublishJobStatus.Failed && x.CreatedAt >= effectiveFrom); if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value); total += await query.CountAsync(cancellationToken); rows.AddRange(await query .OrderByDescending(x => x.CompletedAt ?? x.UpdatedAt) .Take(take) .Select(x => new FailedBackgroundJobItem( x.Id, "ExamPublish", x.Kind == ExamPublishJobKind.FormalExam ? "正式考试发布" : "补考发布", x.Kind == ExamPublishJobKind.FormalExam ? "正式考试计划" : "补考计划", x.ErrorMessage ?? "任务失败但未记录错误详情。", x.CreatedAt, x.StartedAt, x.CompletedAt, 0)) .ToListAsync(cancellationToken)); } var pageItems = rows .OrderByDescending(x => x.CompletedAt ?? x.CreatedAt) .Skip((page - 1) * pageSize) .Take(pageSize) .ToArray(); if (pageItems.Length > 0) { var ids = pageItems.Select(x => x.Id).ToArray(); var attempts = await db.BackgroundJobOutboxMessages.AsNoTracking() .Where(x => ids.Contains(x.JobId)) .Select(x => new { x.JobId, x.ProcessingAttempts }) .ToDictionaryAsync(x => x.JobId, x => x.ProcessingAttempts, cancellationToken); pageItems = pageItems .Select(x => x with { ProcessingAttempts = attempts.GetValueOrDefault(x.Id) }) .ToArray(); } return Ok(new PagedResult( pageItems, total, page, pageSize)); } [HttpGet("backups")] public async Task>> GetBackups( CancellationToken cancellationToken) => Ok(await backupService.ListAsync(cancellationToken)); [HttpPost("backups")] public async Task> CreateBackup( CreateBackupRequest request, CancellationToken cancellationToken) { try { var artifact = await backupService.CreateAsync( request.Note, cancellationToken); return CreatedAtAction( nameof(GetBackups), new { id = artifact.Id }, artifact); } catch (Exception exception) when (exception is not OperationCanceledException) { return Problem( title: "数据库备份失败", detail: SafeMessage(exception), statusCode: StatusCodes.Status503ServiceUnavailable); } } [HttpPost("backups/{backupId}/restore-drill")] public async Task> RunRestoreDrill( string backupId, RestoreDrillRequest request, CancellationToken cancellationToken) { if (!request.Confirmation.Equals( "RESTORE_DRILL", StringComparison.Ordinal)) { return ValidationProblem( "恢复演练必须明确确认,且不会覆盖当前业务数据库。"); } try { return Ok(await backupService.RunRestoreDrillAsync( backupId, cancellationToken)); } catch (FileNotFoundException) { return NotFound(new ProblemDetails { Title = "备份不存在", Detail = "指定备份不存在或其文件已被移除。", Status = StatusCodes.Status404NotFound }); } } private async Task> BuildAlertsAsync( OperationalHealthSnapshot health, IReadOnlyCollection backups, int serverErrorCount, int failedJobCount, CancellationToken cancellationToken) { var alerts = health.Components .Where(x => x.Status != "healthy") .Select(x => new OperationalAlert( $"health-{x.Key}", x.Status == "unhealthy" ? "critical" : "warning", "health", $"{x.Label}状态异常", x.Detail, health.CheckedAt)) .ToList(); if (serverErrorCount > 0) { var latest = await db.AuditLogs.AsNoTracking() .Where(x => x.CreatedAt >= DateTime.UtcNow.AddHours(-24) && x.StatusCode >= 500) .OrderByDescending(x => x.CreatedAt) .Select(x => new { x.Path, x.StatusCode, x.CreatedAt }) .FirstOrDefaultAsync(cancellationToken); alerts.Add(new OperationalAlert( "http-5xx", "critical", "audit", $"过去 24 小时发生 {serverErrorCount} 次服务端错误", latest is null ? "请检查服务日志定位异常。" : $"最近一次为 {latest.StatusCode} {latest.Path}。", latest?.CreatedAt ?? DateTime.UtcNow)); } if (failedJobCount > 0) { alerts.Add(new OperationalAlert( "failed-jobs", "critical", "jobs", $"最近 7 天有 {failedJobCount} 个后台任务失败", "任务已停止或达到重试上限,请在失败任务中查看错误详情。", DateTime.UtcNow)); } var retryingFailures = await db.BackgroundJobOutboxMessages.AsNoTracking() .CountAsync( x => x.State != BackgroundJobOutboxState.Completed && x.LastError != null, cancellationToken); if (retryingFailures > 0) { alerts.Add(new OperationalAlert( "retrying-jobs", "warning", "jobs", $"{retryingFailures} 个后台任务正在错误重试", "任务队列仍会自动重试;若持续出现,请检查依赖服务与任务参数。", DateTime.UtcNow)); } var latestBackup = backups.FirstOrDefault(); if (latestBackup is null) { alerts.Add(new OperationalAlert( "backup-missing", "critical", "backup", "尚无可验证的数据库备份", "立即创建首个备份,并在创建后执行一次隔离恢复演练。", DateTime.UtcNow)); } else { var ageHours = (DateTime.UtcNow - latestBackup.CreatedAt).TotalHours; if (ageHours > options.BackupWarningHours) { alerts.Add(new OperationalAlert( "backup-stale", "warning", "backup", $"最近备份已超过 {options.BackupWarningHours} 小时", $"最近备份创建于 {latestBackup.CreatedAt:u}。", latestBackup.CreatedAt)); } if (latestBackup.LastDrillSucceeded == false) { alerts.Add(new OperationalAlert( "restore-drill-failed", "critical", "backup", "最近一次恢复演练失败", latestBackup.LastDrillDetail ?? "请重新运行演练并检查数据库工具日志。", latestBackup.LastDrillAt ?? latestBackup.CreatedAt)); } else if (!latestBackup.LastDrillAt.HasValue) { alerts.Add(new OperationalAlert( "restore-drill-missing", "warning", "backup", "最近备份尚未完成恢复演练", "恢复演练只写入隔离数据库,不会覆盖当前业务数据。", latestBackup.CreatedAt)); } } return alerts .OrderBy(x => x.Severity == "critical" ? 0 : 1) .ThenByDescending(x => x.OccurredAt) .ToArray(); } private async Task CountFailedJobsAsync( DateTime from, CancellationToken cancellationToken) => await db.AutomaticScheduleJobs.AsNoTracking() .CountAsync( x => x.Status == AutomaticScheduleJobStatus.Failed && x.CreatedAt >= from, cancellationToken) + await db.SchedulePublishJobs.AsNoTracking() .CountAsync( x => x.Status == SchedulePublishJobStatus.Failed && x.CreatedAt >= from, cancellationToken) + await db.MakeupExamAutoJobs.AsNoTracking() .CountAsync( x => x.Status == MakeupExamAutoJobStatus.Failed && x.CreatedAt >= from, cancellationToken) + await db.ExamArrangementJobs.AsNoTracking() .CountAsync( x => x.Status == ExamArrangementJobStatus.Failed && x.CreatedAt >= from, cancellationToken) + await db.ExamPublishJobs.AsNoTracking() .CountAsync( x => x.Status == ExamPublishJobStatus.Failed && x.CreatedAt >= from, cancellationToken); private ActionResult? ValidatePaging(int page, int pageSize) { if (page is < 1 or > 100000 || pageSize is < 1 or > 100) { return ValidationProblem( "页码必须在 1 到 100000 之间,每页数量必须在 1 到 100 之间。"); } return null; } private ActionResult? ValidateRange(DateTime? from, DateTime? to) { if (from.HasValue && to.HasValue && from.Value > to.Value) return ValidationProblem("开始时间不能晚于结束时间。"); return null; } private static string? NormalizeJobKind(string? kind) { if (string.IsNullOrWhiteSpace(kind)) return null; return kind.Trim() switch { "AutomaticSchedule" => "AutomaticSchedule", "SchedulePublish" => "SchedulePublish", "MakeupExamAuto" => "MakeupExamAuto", "ExamArrangement" => "ExamArrangement", _ => null }; } private static string SafeMessage(Exception exception) { var message = exception.GetBaseException().Message; return message.Length <= 500 ? message : message[..500]; } } public sealed record AuditLogItem( Guid Id, string? UserName, string Method, string Path, int StatusCode, string? IpAddress, DateTime CreatedAt); public sealed record FailedBackgroundJobItem( Guid Id, string Kind, string KindLabel, string Context, string ErrorMessage, DateTime CreatedAt, DateTime? StartedAt, DateTime? CompletedAt, int ProcessingAttempts); public sealed record OperationalAlert( string Id, string Severity, string Source, string Title, string Detail, DateTime OccurredAt); public sealed record OperationsCounters( int AuditEvents24Hours, int ServerErrors24Hours, int FailedJobs7Days, int CriticalAlerts); public sealed record OperationsSummary( DateTime GeneratedAt, OperationalHealthSnapshot Health, OperationsCounters Counters, IReadOnlyCollection Alerts, BackupArtifact? LatestBackup); public sealed record CreateBackupRequest([MaxLength(200)] string? Note); public sealed record RestoreDrillRequest([Required] string Confirmation);