From 4d8de1e4aee200d377ff8dff4d9eac063ab8eb36 Mon Sep 17 00:00:00 2001 From: biss Date: Mon, 27 Jul 2026 16:37:45 +0800 Subject: [PATCH] =?UTF-8?q?=E2=80=9C=E8=BF=90=E7=BB=B4=E4=B8=8E=E5=AE=A1?= =?UTF-8?q?=E8=AE=A1=E6=8E=A7=E5=88=B6=E5=8F=B0=E2=80=9D=E3=80=82=20?= =?UTF-8?q?=E4=B8=BB=E8=A6=81=E8=83=BD=E5=8A=9B=EF=BC=9A=20SuperAdmin=20?= =?UTF-8?q?=E4=B8=93=E7=94=A8=E5=85=A5=E5=8F=A3=EF=BC=9A=E7=BB=84=E7=BB=87?= =?UTF-8?q?=E4=B8=8E=E6=9D=83=E9=99=90=20=E2=86=92=20=E8=BF=90=E7=BB=B4?= =?UTF-8?q?=E4=B8=8E=E5=AE=A1=E8=AE=A1=E3=80=82=20=E6=93=8D=E4=BD=9C?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=88=86=E9=A1=B5=E6=9F=A5=E8=AF=A2=EF=BC=8C?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E6=97=B6=E9=97=B4=E3=80=81=E8=B4=A6=E5=8F=B7?= =?UTF-8?q?=E3=80=81=E8=B7=AF=E5=BE=84=E3=80=81=E6=96=B9=E6=B3=95=E5=92=8C?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E7=A0=81=E7=AD=9B=E9=80=89=E3=80=82=20?= =?UTF-8?q?=E6=B1=87=E6=80=BB=E8=87=AA=E5=8A=A8=E6=8E=92=E8=AF=BE=E3=80=81?= =?UTF-8?q?=E8=AF=BE=E8=A1=A8=E5=8F=91=E5=B8=83=E3=80=81=E8=A1=A5=E8=80=83?= =?UTF-8?q?=E5=AE=89=E6=8E=92=E4=B8=89=E7=B1=BB=E5=A4=B1=E8=B4=A5=E5=90=8E?= =?UTF-8?q?=E5=8F=B0=E4=BB=BB=E5=8A=A1=E3=80=82=20=E5=AE=9E=E6=97=B6?= =?UTF-8?q?=E6=A3=80=E6=9F=A5=E6=95=B0=E6=8D=AE=E5=BA=93=E3=80=81=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E3=80=81=E4=BB=BB=E5=8A=A1=E9=80=9A=E9=81=93=E5=8F=8A?= =?UTF-8?q?=E7=A7=AF=E5=8E=8B=E7=8A=B6=E6=80=81=E3=80=82=20=E8=81=9A?= =?UTF-8?q?=E5=90=88=205xx=E3=80=81=E5=A4=B1=E8=B4=A5/=E9=87=8D=E8=AF=95?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E3=80=81=E5=81=A5=E5=BA=B7=E6=8E=A2=E9=92=88?= =?UTF-8?q?=E5=92=8C=E5=A4=87=E4=BB=BD=E6=97=B6=E6=95=88=E5=91=8A=E8=AD=A6?= =?UTF-8?q?=E3=80=82=20SQLite=20=E5=9C=A8=E7=BA=BF=E5=A4=87=E4=BB=BD?= =?UTF-8?q?=EF=BC=9BMySQL=20=E8=B0=83=E7=94=A8=E5=8E=9F=E7=94=9F=E5=AE=A2?= =?UTF-8?q?=E6=88=B7=E7=AB=AF=E5=A4=87=E4=BB=BD=E3=80=82=20SHA-256=20?= =?UTF-8?q?=E6=A0=A1=E9=AA=8C=E5=8F=8A=E9=9A=94=E7=A6=BB=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E6=81=A2=E5=A4=8D=E6=BC=94=E7=BB=83=EF=BC=8C=E4=B8=8D?= =?UTF-8?q?=E8=A6=86=E7=9B=96=E4=B8=9A=E5=8A=A1=E5=BA=93=E3=80=82=20MySQL?= =?UTF-8?q?=20=E5=BC=BA=E5=88=B6=E4=BD=BF=E7=94=A8=E7=8B=AC=E7=AB=8B?= =?UTF-8?q?=E8=BF=90=E7=BB=B4=E8=BF=9E=E6=8E=A5=EF=BC=8C=E5=AE=B9=E5=99=A8?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=8C=81=E4=B9=85=E5=8C=96=E5=A4=87=E4=BB=BD?= =?UTF-8?q?=E5=8D=B7=E4=B8=8E=E6=95=B0=E6=8D=AE=E5=BA=93=E5=AE=A2=E6=88=B7?= =?UTF-8?q?=E7=AB=AF=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 13 + Dockerfile | 11 +- README.md | 25 + compose.app.example.yml | 8 +- compose.example.yml | 3 + .../Controllers/OperationsController.cs | 519 +++++++++ .../Operations/DatabaseBackupService.cs | 584 ++++++++++ .../Operations/OperationalHealthService.cs | 200 ++++ .../Operations/OperationsOptions.cs | 13 + src/Jiaowu.Api/Program.cs | 21 + src/Jiaowu.Api/appsettings.json | 8 + .../OperationsControllerTests.cs | 252 ++++ web/src/layouts/AdminLayout.vue | 1 + web/src/router/index.ts | 6 + web/src/views/OperationsConsoleView.vue | 1027 +++++++++++++++++ 15 files changed, 2687 insertions(+), 4 deletions(-) create mode 100644 src/Jiaowu.Api/Controllers/OperationsController.cs create mode 100644 src/Jiaowu.Api/Infrastructure/Operations/DatabaseBackupService.cs create mode 100644 src/Jiaowu.Api/Infrastructure/Operations/OperationalHealthService.cs create mode 100644 src/Jiaowu.Api/Infrastructure/Operations/OperationsOptions.cs create mode 100644 tests/Jiaowu.Api.Tests/OperationsControllerTests.cs create mode 100644 web/src/views/OperationsConsoleView.vue diff --git a/.env.example b/.env.example index dfc5197..0c33c9d 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,9 @@ Database__Provider=MySql Database__ApplyMigrationsOnStartup=false Database__CommandTimeoutSeconds=30 ConnectionStrings__MySql="Server=db.example.edu.cn;Port=3306;Database=jiaowu;User=APP_USER;Password=REPLACE_WITH_A_STRONG_PASSWORD;SslMode=VerifyFull;SslCa=/etc/jiaowu/mysql-ca.pem;" +# 仅供备份与隔离恢复演练使用。该账号需要读取业务库，并仅能创建/删除 +# jiaowu_restore_drill_* 临时库；不要在此复用日常业务账号。 +# ConnectionStrings__OperationsMySql="Server=db.example.edu.cn;Port=3306;Database=jiaowu;User=OPS_USER;Password=REPLACE_WITH_A_STRONG_PASSWORD;SslMode=VerifyFull;SslCa=/etc/jiaowu/mysql-ca.pem;" # Redis 是可选加速器；留空时应用仅使用进程内缓存。 # ConnectionStrings__Redis="redis.example.edu.cn:6380,user=jiaowu,password=REPLACE_WITH_A_STRONG_PASSWORD,ssl=true,abortConnect=false" @@ -32,6 +35,16 @@ Cache__AnalyticsExpirationMinutes=3 Cache__AnalyticsLocalExpirationSeconds=30 Cache__MaximumPayloadKilobytes=2048 +# 运维控制台备份目录必须位于持久化、仅服务账号可写的位置。 +Operations__BackupDirectory=/var/lib/jiaowu/backups +Operations__BackupWarningHours=24 +Operations__ToolTimeoutMinutes=30 +Operations__MySqlDumpPath=mysqldump +Operations__MySqlClientPath=mysql +# 按实际客户端补充 TLS 参数；Oracle MySQL 客户端示例： +# Operations__MySqlAdditionalArguments__0=--ssl-mode=VERIFY_IDENTITY +# Operations__MySqlAdditionalArguments__1=--ssl-ca=/etc/jiaowu/mysql-ca.pem + Jwt__Issuer=Jiaowu.Api Jwt__Audience=Jiaowu.Web Jwt__Key=REPLACE_WITH_AT_LEAST_32_RANDOM_BYTES diff --git a/Dockerfile b/Dockerfile index e00893c..10b4ff5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,11 +30,14 @@ FROM mcr.microsoft.com/dotnet/aspnet:10.0-alpine AS final WORKDIR /app ENV ASPNETCORE_ENVIRONMENT=Production \ ASPNETCORE_HTTP_PORTS=8080 \ - DOTNET_EnableDiagnostics=0 + DOTNET_EnableDiagnostics=0 \ + Operations__BackupDirectory=/var/lib/jiaowu/backups \ + Operations__MySqlDumpPath=mariadb-dump \ + Operations__MySqlClientPath=mariadb EXPOSE 8080 COPY --from=build /app/publish/ ./ ARG UID=10001 -RUN apk add --no-cache font-noto-cjk +RUN apk add --no-cache font-noto-cjk mariadb-client RUN adduser \ --disabled-password \ --gecos "" \ @@ -43,5 +46,9 @@ RUN adduser \ --no-create-home \ --uid "${UID}" \ appuser +RUN mkdir -p /var/lib/jiaowu/backups \ + && chown appuser:appuser /var/lib/jiaowu/backups \ + && chmod 700 /var/lib/jiaowu/backups +VOLUME ["/var/lib/jiaowu/backups"] USER appuser ENTRYPOINT ["dotnet", "Jiaowu.Api.dll"] diff --git a/README.md b/README.md index 05831d2..6d37b1f 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,12 @@ sudo chown -R root:jiaowu /opt/jiaowu sudo chmod 0750 /opt/jiaowu sudo chmod 0750 /opt/jiaowu/Jiaowu.Api sudo chmod 0640 /opt/jiaowu/.env +sudo install -d \ + --owner=jiaowu \ + --group=jiaowu \ + --mode=0700 \ + /var/lib/jiaowu/backups +sudo apt-get install default-mysql-client ``` 如果账号已存在，`useradd` 会报错，可以跳过该命令。RHEL 系发行版的 `nologin` 通常 @@ -297,6 +303,25 @@ Outbox 租约恢复改为按维护周期执行，避免积压发布时每条消 消息会根据 Outbox 状态和租约继续补投。迁移服务应先应用 `BackgroundJobOutbox` 数据库迁移，再启动应用实例。 +### 运维与审计控制台 + +超级管理员可从“组织与权限 → 运维与审计”查询写操作日志、三类失败后台任务、数据库、 +缓存与任务通道健康状态，并查看由 5xx、失败/重试任务、健康探针和备份时效汇总出的异常 +告警。查询接口和备份操作均在后端强制要求 `SuperAdmin`，不能只依赖前端菜单隐藏。 + +SQLite 开发环境直接使用在线备份 API。MySQL 环境需要在服务器安装 `mysqldump` 与 +`mysql`（容器镜像已包含对应的 `mariadb-dump` 与 `mariadb` 客户端），并配置独立的 +`ConnectionStrings__OperationsMySql`。该账号不得复用日常业务账号：它需要读取业务库， +并只应被授权创建和删除名称为 `jiaowu_restore_drill_*` 的临时演练库。恢复演练不会覆盖 +当前业务库，流程是“校验 SHA-256 → 恢复到随机临时库 → 检查表结构 → 删除临时库”。 + +备份目录必须是仅服务账号可写的持久化目录。示例配置使用 +`/var/lib/jiaowu/backups`；Compose 已挂载独立命名卷。启用 MySQL TLS 时，还要通过 +`Operations__MySqlAdditionalArguments__N` 传入与所选命令行客户端匹配的 CA 与主机名 +校验参数。例如 Oracle MySQL 客户端使用 `--ssl-mode=VERIFY_IDENTITY` 和 +`--ssl-ca=/etc/jiaowu/mysql-ca.pem`，容器内 MariaDB 客户端使用 `--ssl`、 +`--ssl-ca=...` 与 `--ssl-verify-server-cert`。 + ## 跨平台发布与 Docker `.gitea/workflows/publish.yml` 只在推送 `v*` 标签或手动运行时执行，普通分支 push diff --git a/compose.app.example.yml b/compose.app.example.yml index 5332709..271f527 100644 --- a/compose.app.example.yml +++ b/compose.app.example.yml @@ -17,11 +17,15 @@ services: options: max-size: "10m" max-file: "3" + volumes: + - jiaowu-backups:/var/lib/jiaowu/backups # 如果 .env 中的 SslCa=/etc/jiaowu/mysql-ca.pem，请把 CA 放到 - # ./certs/mysql-ca.pem，并取消下面三行注释。 - # volumes: + # ./certs/mysql-ca.pem，并在上面的 volumes 中追加以下四行。 # - type: bind # source: ./certs/mysql-ca.pem # target: /etc/jiaowu/mysql-ca.pem # read_only: true + +volumes: + jiaowu-backups: diff --git a/compose.example.yml b/compose.example.yml index fce0a6e..79ec651 100644 --- a/compose.example.yml +++ b/compose.example.yml @@ -138,6 +138,8 @@ services: - "${JIAOWU_PORT:-8080}:8080" restart: unless-stopped init: true + volumes: + - backup-data:/var/lib/jiaowu/backups logging: *json-logging # 工具型一次性服务：普通 docker compose up 不会执行它。 @@ -161,3 +163,4 @@ services: volumes: mysql-data: rabbitmq-data: + backup-data: diff --git a/src/Jiaowu.Api/Controllers/OperationsController.cs b/src/Jiaowu.Api/Controllers/OperationsController.cs new file mode 100644 index 0000000..e16946e --- /dev/null +++ b/src/Jiaowu.Api/Controllers/OperationsController.cs @@ -0,0 +1,519 @@ +using System.ComponentModel.DataAnnotations; +using Jiaowu.Api.Contracts; +using Jiaowu.Api.Domain.Academic; +using Jiaowu.Api.Domain.Identity; +using Jiaowu.Api.Domain.System; +using Jiaowu.Api.Infrastructure.Operations; +using Jiaowu.Api.Infrastructure.Persistence; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; + +namespace Jiaowu.Api.Controllers; + +[ApiController] +[Authorize(Roles = SystemRoles.SuperAdmin)] +[Route("api/operations")] +public sealed class OperationsController( + AppDbContext db, + OperationalHealthService healthService, + DatabaseBackupService backupService, + OperationsOptions options) : ControllerBase +{ + [HttpGet("summary")] + public async Task> GetSummary( + CancellationToken cancellationToken) + { + var since = DateTime.UtcNow.AddHours(-24); + var health = await healthService.CheckAsync(cancellationToken); + var backups = await backupService.ListAsync(cancellationToken); + var auditCount = await db.AuditLogs.AsNoTracking() + .CountAsync(x => x.CreatedAt >= since, cancellationToken); + var serverErrorCount = await db.AuditLogs.AsNoTracking() + .CountAsync( + x => x.CreatedAt >= since && x.StatusCode >= 500, + cancellationToken); + var failedJobCount = await CountFailedJobsAsync( + DateTime.UtcNow.AddDays(-7), + cancellationToken); + var alerts = await BuildAlertsAsync( + health, + backups, + serverErrorCount, + failedJobCount, + cancellationToken); + return Ok(new OperationsSummary( + DateTime.UtcNow, + health, + new OperationsCounters( + auditCount, + serverErrorCount, + failedJobCount, + alerts.Count(x => x.Severity == "critical")), + alerts, + backups.FirstOrDefault())); + } + + [HttpGet("health")] + public async Task> GetHealth( + CancellationToken cancellationToken) => + Ok(await healthService.CheckAsync(cancellationToken)); + + [HttpGet("audit-logs")] + public async Task>> GetAuditLogs( + [FromQuery] int page = 1, + [FromQuery] int pageSize = 20, + [FromQuery] string? method = null, + [FromQuery] int? statusCode = null, + [FromQuery] string? userName = null, + [FromQuery] string? path = null, + [FromQuery] DateTime? from = null, + [FromQuery] DateTime? to = null, + CancellationToken cancellationToken = default) + { + var pagingError = ValidatePaging(page, pageSize); + if (pagingError is not null) return pagingError; + var rangeError = ValidateRange(from, to); + if (rangeError is not null) return rangeError; + + var query = db.AuditLogs.AsNoTracking().AsQueryable(); + if (!string.IsNullOrWhiteSpace(method)) + { + var normalizedMethod = method.Trim().ToUpperInvariant(); + query = query.Where(x => x.Method == normalizedMethod); + } + if (statusCode.HasValue) + query = query.Where(x => x.StatusCode == statusCode.Value); + if (!string.IsNullOrWhiteSpace(userName)) + { + var normalizedUser = userName.Trim(); + query = query.Where(x => + x.UserName != null && x.UserName.Contains(normalizedUser)); + } + if (!string.IsNullOrWhiteSpace(path)) + { + var normalizedPath = path.Trim(); + query = query.Where(x => x.Path.Contains(normalizedPath)); + } + query = query.Where(x => + x.CreatedAt >= (from ?? DateTime.UtcNow.AddDays(-1))); + if (to.HasValue) + query = query.Where(x => x.CreatedAt <= to.Value); + + var total = await query.CountAsync(cancellationToken); + var items = await query + .OrderByDescending(x => x.CreatedAt) + .Skip((page - 1) * pageSize) + .Take(pageSize) + .Select(x => new AuditLogItem( + x.Id, + x.UserName, + x.Method, + x.Path, + x.StatusCode, + x.IpAddress, + x.CreatedAt)) + .ToListAsync(cancellationToken); + return Ok(new PagedResult(items, total, page, pageSize)); + } + + [HttpGet("failed-jobs")] + public async Task>> + GetFailedJobs( + [FromQuery] int page = 1, + [FromQuery] int pageSize = 20, + [FromQuery] string? kind = null, + [FromQuery] DateTime? from = null, + [FromQuery] DateTime? to = null, + CancellationToken cancellationToken = default) + { + var pagingError = ValidatePaging(page, pageSize); + if (pagingError is not null) return pagingError; + var rangeError = ValidateRange(from, to); + if (rangeError is not null) return rangeError; + + var normalizedKind = NormalizeJobKind(kind); + if (kind is not null && normalizedKind is null) + return ValidationProblem("后台任务类型无效。"); + + var effectiveFrom = from ?? DateTime.UtcNow.AddDays(-30); + var take = checked(page * pageSize); + var rows = new List(); + var total = 0; + + if (normalizedKind is null or "AutomaticSchedule") + { + var query = db.AutomaticScheduleJobs.AsNoTracking() + .Where(x => + x.Status == AutomaticScheduleJobStatus.Failed && + x.CreatedAt >= effectiveFrom); + if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value); + total += await query.CountAsync(cancellationToken); + rows.AddRange(await query + .OrderByDescending(x => x.CompletedAt ?? x.UpdatedAt) + .Take(take) + .Select(x => new FailedBackgroundJobItem( + x.Id, + "AutomaticSchedule", + "自动排课", + x.SchedulePlan == null ? "排课方案" : x.SchedulePlan.Name, + x.ErrorMessage ?? "任务失败但未记录错误详情。", + x.CreatedAt, + x.StartedAt, + x.CompletedAt, + 0)) + .ToListAsync(cancellationToken)); + } + + if (normalizedKind is null or "SchedulePublish") + { + var query = db.SchedulePublishJobs.AsNoTracking() + .Where(x => + x.Status == SchedulePublishJobStatus.Failed && + x.CreatedAt >= effectiveFrom); + if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value); + total += await query.CountAsync(cancellationToken); + rows.AddRange(await query + .OrderByDescending(x => x.CompletedAt ?? x.UpdatedAt) + .Take(take) + .Select(x => new FailedBackgroundJobItem( + x.Id, + "SchedulePublish", + "课表发布", + x.SchedulePlan == null ? "排课方案" : x.SchedulePlan.Name, + x.ErrorMessage ?? "任务失败但未记录错误详情。", + x.CreatedAt, + x.StartedAt, + x.CompletedAt, + 0)) + .ToListAsync(cancellationToken)); + } + + if (normalizedKind is null or "MakeupExamAuto") + { + var query = db.MakeupExamAutoJobs.AsNoTracking() + .Where(x => + x.Status == MakeupExamAutoJobStatus.Failed && + x.CreatedAt >= effectiveFrom); + if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value); + total += await query.CountAsync(cancellationToken); + rows.AddRange(await query + .OrderByDescending(x => x.CompletedAt ?? x.UpdatedAt) + .Take(take) + .Select(x => new FailedBackgroundJobItem( + x.Id, + "MakeupExamAuto", + "补考自动安排", + x.MakeupExamPlan == null ? "补考计划" : x.MakeupExamPlan.Name, + x.ErrorMessage ?? "任务失败但未记录错误详情。", + x.CreatedAt, + x.StartedAt, + x.CompletedAt, + 0)) + .ToListAsync(cancellationToken)); + } + + var pageItems = rows + .OrderByDescending(x => x.CompletedAt ?? x.CreatedAt) + .Skip((page - 1) * pageSize) + .Take(pageSize) + .ToArray(); + if (pageItems.Length > 0) + { + var ids = pageItems.Select(x => x.Id).ToArray(); + var attempts = await db.BackgroundJobOutboxMessages.AsNoTracking() + .Where(x => ids.Contains(x.JobId)) + .Select(x => new { x.JobId, x.ProcessingAttempts }) + .ToDictionaryAsync(x => x.JobId, x => x.ProcessingAttempts, + cancellationToken); + pageItems = pageItems + .Select(x => x with + { + ProcessingAttempts = attempts.GetValueOrDefault(x.Id) + }) + .ToArray(); + } + + return Ok(new PagedResult( + pageItems, + total, + page, + pageSize)); + } + + [HttpGet("backups")] + public async Task>> GetBackups( + CancellationToken cancellationToken) => + Ok(await backupService.ListAsync(cancellationToken)); + + [HttpPost("backups")] + public async Task> CreateBackup( + CreateBackupRequest request, + CancellationToken cancellationToken) + { + try + { + var artifact = await backupService.CreateAsync( + request.Note, + cancellationToken); + return CreatedAtAction( + nameof(GetBackups), + new { id = artifact.Id }, + artifact); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + return Problem( + title: "数据库备份失败", + detail: SafeMessage(exception), + statusCode: StatusCodes.Status503ServiceUnavailable); + } + } + + [HttpPost("backups/{backupId}/restore-drill")] + public async Task> RunRestoreDrill( + string backupId, + RestoreDrillRequest request, + CancellationToken cancellationToken) + { + if (!request.Confirmation.Equals( + "RESTORE_DRILL", + StringComparison.Ordinal)) + { + return ValidationProblem( + "恢复演练必须明确确认，且不会覆盖当前业务数据库。"); + } + + try + { + return Ok(await backupService.RunRestoreDrillAsync( + backupId, + cancellationToken)); + } + catch (FileNotFoundException) + { + return NotFound(new ProblemDetails + { + Title = "备份不存在", + Detail = "指定备份不存在或其文件已被移除。", + Status = StatusCodes.Status404NotFound + }); + } + } + + private async Task> BuildAlertsAsync( + OperationalHealthSnapshot health, + IReadOnlyCollection backups, + int serverErrorCount, + int failedJobCount, + CancellationToken cancellationToken) + { + var alerts = health.Components + .Where(x => x.Status != "healthy") + .Select(x => new OperationalAlert( + $"health-{x.Key}", + x.Status == "unhealthy" ? "critical" : "warning", + "health", + $"{x.Label}状态异常", + x.Detail, + health.CheckedAt)) + .ToList(); + + if (serverErrorCount > 0) + { + var latest = await db.AuditLogs.AsNoTracking() + .Where(x => + x.CreatedAt >= DateTime.UtcNow.AddHours(-24) && + x.StatusCode >= 500) + .OrderByDescending(x => x.CreatedAt) + .Select(x => new { x.Path, x.StatusCode, x.CreatedAt }) + .FirstOrDefaultAsync(cancellationToken); + alerts.Add(new OperationalAlert( + "http-5xx", + "critical", + "audit", + $"过去 24 小时发生 {serverErrorCount} 次服务端错误", + latest is null + ? "请检查服务日志定位异常。" + : $"最近一次为 {latest.StatusCode} {latest.Path}。", + latest?.CreatedAt ?? DateTime.UtcNow)); + } + + if (failedJobCount > 0) + { + alerts.Add(new OperationalAlert( + "failed-jobs", + "critical", + "jobs", + $"最近 7 天有 {failedJobCount} 个后台任务失败", + "任务已停止或达到重试上限，请在失败任务中查看错误详情。", + DateTime.UtcNow)); + } + + var retryingFailures = await db.BackgroundJobOutboxMessages.AsNoTracking() + .CountAsync( + x => x.State != BackgroundJobOutboxState.Completed && + x.LastError != null, + cancellationToken); + if (retryingFailures > 0) + { + alerts.Add(new OperationalAlert( + "retrying-jobs", + "warning", + "jobs", + $"{retryingFailures} 个后台任务正在错误重试", + "任务队列仍会自动重试；若持续出现，请检查依赖服务与任务参数。", + DateTime.UtcNow)); + } + + var latestBackup = backups.FirstOrDefault(); + if (latestBackup is null) + { + alerts.Add(new OperationalAlert( + "backup-missing", + "critical", + "backup", + "尚无可验证的数据库备份", + "立即创建首个备份，并在创建后执行一次隔离恢复演练。", + DateTime.UtcNow)); + } + else + { + var ageHours = (DateTime.UtcNow - latestBackup.CreatedAt).TotalHours; + if (ageHours > options.BackupWarningHours) + { + alerts.Add(new OperationalAlert( + "backup-stale", + "warning", + "backup", + $"最近备份已超过 {options.BackupWarningHours} 小时", + $"最近备份创建于 {latestBackup.CreatedAt:u}。", + latestBackup.CreatedAt)); + } + if (latestBackup.LastDrillSucceeded == false) + { + alerts.Add(new OperationalAlert( + "restore-drill-failed", + "critical", + "backup", + "最近一次恢复演练失败", + latestBackup.LastDrillDetail ?? "请重新运行演练并检查数据库工具日志。", + latestBackup.LastDrillAt ?? latestBackup.CreatedAt)); + } + else if (!latestBackup.LastDrillAt.HasValue) + { + alerts.Add(new OperationalAlert( + "restore-drill-missing", + "warning", + "backup", + "最近备份尚未完成恢复演练", + "恢复演练只写入隔离数据库，不会覆盖当前业务数据。", + latestBackup.CreatedAt)); + } + } + + return alerts + .OrderBy(x => x.Severity == "critical" ? 0 : 1) + .ThenByDescending(x => x.OccurredAt) + .ToArray(); + } + + private async Task CountFailedJobsAsync( + DateTime from, + CancellationToken cancellationToken) => + await db.AutomaticScheduleJobs.AsNoTracking() + .CountAsync( + x => x.Status == AutomaticScheduleJobStatus.Failed && + x.CreatedAt >= from, + cancellationToken) + + await db.SchedulePublishJobs.AsNoTracking() + .CountAsync( + x => x.Status == SchedulePublishJobStatus.Failed && + x.CreatedAt >= from, + cancellationToken) + + await db.MakeupExamAutoJobs.AsNoTracking() + .CountAsync( + x => x.Status == MakeupExamAutoJobStatus.Failed && + x.CreatedAt >= from, + cancellationToken); + + private ActionResult? ValidatePaging(int page, int pageSize) + { + if (page is < 1 or > 100000 || pageSize is < 1 or > 100) + { + return ValidationProblem( + "页码必须在 1 到 100000 之间，每页数量必须在 1 到 100 之间。"); + } + return null; + } + + private ActionResult? ValidateRange(DateTime? from, DateTime? to) + { + if (from.HasValue && to.HasValue && from.Value > to.Value) + return ValidationProblem("开始时间不能晚于结束时间。"); + return null; + } + + private static string? NormalizeJobKind(string? kind) + { + if (string.IsNullOrWhiteSpace(kind)) return null; + return kind.Trim() switch + { + "AutomaticSchedule" => "AutomaticSchedule", + "SchedulePublish" => "SchedulePublish", + "MakeupExamAuto" => "MakeupExamAuto", + _ => null + }; + } + + private static string SafeMessage(Exception exception) + { + var message = exception.GetBaseException().Message; + return message.Length <= 500 ? message : message[..500]; + } +} + +public sealed record AuditLogItem( + Guid Id, + string? UserName, + string Method, + string Path, + int StatusCode, + string? IpAddress, + DateTime CreatedAt); + +public sealed record FailedBackgroundJobItem( + Guid Id, + string Kind, + string KindLabel, + string Context, + string ErrorMessage, + DateTime CreatedAt, + DateTime? StartedAt, + DateTime? CompletedAt, + int ProcessingAttempts); + +public sealed record OperationalAlert( + string Id, + string Severity, + string Source, + string Title, + string Detail, + DateTime OccurredAt); + +public sealed record OperationsCounters( + int AuditEvents24Hours, + int ServerErrors24Hours, + int FailedJobs7Days, + int CriticalAlerts); + +public sealed record OperationsSummary( + DateTime GeneratedAt, + OperationalHealthSnapshot Health, + OperationsCounters Counters, + IReadOnlyCollection Alerts, + BackupArtifact? LatestBackup); + +public sealed record CreateBackupRequest([MaxLength(200)] string? Note); + +public sealed record RestoreDrillRequest([Required] string Confirmation); diff --git a/src/Jiaowu.Api/Infrastructure/Operations/DatabaseBackupService.cs b/src/Jiaowu.Api/Infrastructure/Operations/DatabaseBackupService.cs new file mode 100644 index 0000000..9a1e7b1 --- /dev/null +++ b/src/Jiaowu.Api/Infrastructure/Operations/DatabaseBackupService.cs @@ -0,0 +1,584 @@ +using System.Diagnostics; +using System.Security.Cryptography; +using System.Text.Json; +using Jiaowu.Api.Infrastructure.Persistence; +using Microsoft.Data.Sqlite; +using MySql.Data.MySqlClient; + +namespace Jiaowu.Api.Infrastructure.Operations; + +public sealed record BackupArtifact( + string Id, + string FileName, + string Provider, + DateTime CreatedAt, + long SizeBytes, + string Sha256, + string? Note, + DateTime? LastDrillAt, + bool? LastDrillSucceeded, + string? LastDrillDetail, + long? LastDrillDurationMilliseconds); + +public sealed record RestoreDrillResult( + string BackupId, + bool Succeeded, + DateTime CompletedAt, + string Detail, + long DurationMilliseconds, + int? TableCount); + +public sealed class DatabaseBackupService( + DatabaseOptions databaseOptions, + OperationsOptions options, + IConfiguration configuration, + IHostEnvironment environment, + ILogger logger) +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true + }; + + private readonly SemaphoreSlim operationLock = new(1, 1); + private readonly string backupDirectory = ResolveBackupDirectory( + options.BackupDirectory, + environment.ContentRootPath); + + public async Task> ListAsync( + CancellationToken cancellationToken) + { + Directory.CreateDirectory(backupDirectory); + var items = new List(); + foreach (var metadataPath in Directory.EnumerateFiles( + backupDirectory, + "*.metadata.json", + SearchOption.TopDirectoryOnly)) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + await using var stream = File.OpenRead(metadataPath); + var artifact = await JsonSerializer.DeserializeAsync( + stream, + JsonOptions, + cancellationToken); + if (artifact is not null && + File.Exists(Path.Combine(backupDirectory, artifact.FileName))) + { + items.Add(artifact); + } + } + catch (Exception exception) when ( + exception is IOException or UnauthorizedAccessException or JsonException) + { + logger.LogWarning( + exception, + "Unable to read backup metadata {MetadataFile}.", + Path.GetFileName(metadataPath)); + } + } + + return items + .OrderByDescending(x => x.CreatedAt) + .ToArray(); + } + + public async Task CreateAsync( + string? note, + CancellationToken cancellationToken) + { + await operationLock.WaitAsync(cancellationToken); + try + { + Directory.CreateDirectory(backupDirectory); + var createdAt = DateTime.UtcNow; + var id = $"{createdAt:yyyyMMddHHmmss}-{Guid.NewGuid():N}"[..29]; + var provider = NormalizeProvider(databaseOptions.Provider); + var extension = provider == "SQLite" ? ".sqlite" : ".sql"; + var fileName = $"jiaowu-{id}{extension}"; + var backupPath = Path.Combine(backupDirectory, fileName); + + try + { + if (provider == "SQLite") + await CreateSqliteBackupAsync(backupPath, cancellationToken); + else + await CreateMySqlBackupAsync(backupPath, cancellationToken); + + var fileInfo = new FileInfo(backupPath); + var artifact = new BackupArtifact( + id, + fileName, + provider, + createdAt, + fileInfo.Length, + await ComputeHashAsync(backupPath, cancellationToken), + NormalizeNote(note), + null, + null, + null, + null); + await WriteMetadataAsync(artifact, cancellationToken); + return artifact; + } + catch + { + if (File.Exists(backupPath)) + File.Delete(backupPath); + throw; + } + } + finally + { + operationLock.Release(); + } + } + + public async Task RunRestoreDrillAsync( + string backupId, + CancellationToken cancellationToken) + { + await operationLock.WaitAsync(cancellationToken); + try + { + var artifact = (await ListAsync(cancellationToken)) + .SingleOrDefault(x => x.Id.Equals(backupId, StringComparison.Ordinal)); + if (artifact is null) + throw new FileNotFoundException("未找到指定备份。"); + + var backupPath = Path.Combine(backupDirectory, artifact.FileName); + var actualHash = await ComputeHashAsync(backupPath, cancellationToken); + if (!CryptographicOperations.FixedTimeEquals( + Convert.FromHexString(artifact.Sha256), + Convert.FromHexString(actualHash))) + { + var damaged = await CompleteDrillAsync( + artifact, + false, + "备份文件校验和不一致，恢复演练已中止。", + 0, + null, + cancellationToken); + return damaged; + } + + var stopwatch = Stopwatch.StartNew(); + RestoreDrillResult result; + try + { + var tableCount = artifact.Provider == "SQLite" + ? await DrillSqliteAsync(backupPath, cancellationToken) + : await DrillMySqlAsync(backupPath, cancellationToken); + stopwatch.Stop(); + result = await CompleteDrillAsync( + artifact, + true, + $"已在隔离数据库完成恢复并通过完整性检查，共发现 {tableCount} 张业务表。", + stopwatch.ElapsedMilliseconds, + tableCount, + cancellationToken); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + stopwatch.Stop(); + logger.LogError( + exception, + "Restore drill failed for backup {BackupId}.", + artifact.Id); + result = await CompleteDrillAsync( + artifact, + false, + $"恢复演练失败：{SafeMessage(exception)}", + stopwatch.ElapsedMilliseconds, + null, + cancellationToken); + } + + return result; + } + finally + { + operationLock.Release(); + } + } + + private async Task CreateSqliteBackupAsync( + string backupPath, + CancellationToken cancellationToken) + { + var sourceBuilder = new SqliteConnectionStringBuilder( + configuration.GetConnectionString("SQLite") + ?? throw new InvalidOperationException("缺少 ConnectionStrings:SQLite。")); + if (!Path.IsPathRooted(sourceBuilder.DataSource)) + { + sourceBuilder.DataSource = Path.GetFullPath( + sourceBuilder.DataSource, + environment.ContentRootPath); + } + + var destinationBuilder = new SqliteConnectionStringBuilder + { + DataSource = backupPath, + Mode = SqliteOpenMode.ReadWriteCreate, + Pooling = false + }; + await using var source = new SqliteConnection(sourceBuilder.ConnectionString); + await using var destination = new SqliteConnection(destinationBuilder.ConnectionString); + await source.OpenAsync(cancellationToken); + await destination.OpenAsync(cancellationToken); + source.BackupDatabase(destination); + } + + private async Task CreateMySqlBackupAsync( + string backupPath, + CancellationToken cancellationToken) + { + var connection = GetMySqlConnectionBuilder(); + var arguments = new List + { + "--protocol=tcp", + $"--host={connection.Server}", + $"--port={connection.Port}", + $"--user={connection.UserID}", + "--single-transaction", + "--quick", + "--routines", + "--triggers", + "--events", + "--hex-blob", + "--default-character-set=utf8mb4" + }; + arguments.AddRange(options.MySqlAdditionalArguments); + arguments.Add(connection.Database); + + await using var output = new FileStream( + backupPath, + FileMode.CreateNew, + FileAccess.Write, + FileShare.None, + 81920, + FileOptions.Asynchronous); + await RunToolAsync( + options.MySqlDumpPath, + arguments, + connection.Password, + standardInput: null, + standardOutput: output, + cancellationToken); + } + + private static async Task DrillSqliteAsync( + string backupPath, + CancellationToken cancellationToken) + { + var drillPath = Path.Combine( + Path.GetDirectoryName(backupPath)!, + $".restore-drill-{Guid.NewGuid():N}.sqlite"); + try + { + var sourceBuilder = new SqliteConnectionStringBuilder + { + DataSource = backupPath, + Mode = SqliteOpenMode.ReadOnly, + Pooling = false + }; + var drillBuilder = new SqliteConnectionStringBuilder + { + DataSource = drillPath, + Mode = SqliteOpenMode.ReadWriteCreate, + Pooling = false + }; + await using var source = new SqliteConnection(sourceBuilder.ConnectionString); + await using var drill = new SqliteConnection(drillBuilder.ConnectionString); + await source.OpenAsync(cancellationToken); + await drill.OpenAsync(cancellationToken); + source.BackupDatabase(drill); + + await using var integrity = drill.CreateCommand(); + integrity.CommandText = "PRAGMA integrity_check;"; + var integrityResult = Convert.ToString( + await integrity.ExecuteScalarAsync(cancellationToken)); + if (!string.Equals(integrityResult, "ok", StringComparison.OrdinalIgnoreCase)) + throw new InvalidDataException($"SQLite 完整性检查返回 {integrityResult ?? "空结果"}。"); + + await using var tables = drill.CreateCommand(); + tables.CommandText = + "SELECT COUNT(*) FROM sqlite_master " + + "WHERE type = 'table' AND name NOT LIKE 'sqlite_%';"; + return Convert.ToInt32(await tables.ExecuteScalarAsync(cancellationToken)); + } + finally + { + if (File.Exists(drillPath)) + File.Delete(drillPath); + if (File.Exists($"{drillPath}-shm")) + File.Delete($"{drillPath}-shm"); + if (File.Exists($"{drillPath}-wal")) + File.Delete($"{drillPath}-wal"); + } + } + + private async Task DrillMySqlAsync( + string backupPath, + CancellationToken cancellationToken) + { + var connection = GetMySqlConnectionBuilder(); + var drillDatabase = $"jiaowu_restore_drill_{DateTime.UtcNow:yyyyMMddHHmmss}_" + + Guid.NewGuid().ToString("N")[..8]; + var adminBuilder = new MySqlConnectionStringBuilder(connection.ConnectionString) + { + Database = "" + }; + + await using var admin = new MySqlConnection(adminBuilder.ConnectionString); + await admin.OpenAsync(cancellationToken); + try + { + await using (var create = admin.CreateCommand()) + { + create.CommandText = + $"CREATE DATABASE `{drillDatabase}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"; + await create.ExecuteNonQueryAsync(cancellationToken); + } + + var arguments = new List + { + "--protocol=tcp", + $"--host={connection.Server}", + $"--port={connection.Port}", + $"--user={connection.UserID}", + "--default-character-set=utf8mb4" + }; + arguments.AddRange(options.MySqlAdditionalArguments); + arguments.Add($"--database={drillDatabase}"); + await using (var input = new FileStream( + backupPath, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + 81920, + FileOptions.Asynchronous)) + { + await RunToolAsync( + options.MySqlClientPath, + arguments, + connection.Password, + input, + standardOutput: null, + cancellationToken); + } + + await using var count = admin.CreateCommand(); + count.CommandText = + "SELECT COUNT(*) FROM information_schema.tables " + + "WHERE table_schema = @schema AND table_type = 'BASE TABLE';"; + count.Parameters.AddWithValue("@schema", drillDatabase); + return Convert.ToInt32(await count.ExecuteScalarAsync(cancellationToken)); + } + finally + { + try + { + await using var drop = admin.CreateCommand(); + drop.CommandText = $"DROP DATABASE IF EXISTS `{drillDatabase}`;"; + await drop.ExecuteNonQueryAsync(CancellationToken.None); + } + catch (Exception exception) + { + logger.LogCritical( + exception, + "Unable to remove isolated restore drill database {DatabaseName}.", + drillDatabase); + } + } + } + + private async Task RunToolAsync( + string executable, + IReadOnlyCollection arguments, + string password, + Stream? standardInput, + Stream? standardOutput, + CancellationToken cancellationToken) + { + var startInfo = new ProcessStartInfo + { + FileName = executable, + UseShellExecute = false, + RedirectStandardError = true, + RedirectStandardInput = standardInput is not null, + RedirectStandardOutput = standardOutput is not null, + CreateNoWindow = true + }; + foreach (var argument in arguments) + startInfo.ArgumentList.Add(argument); + if (!string.IsNullOrEmpty(password)) + startInfo.Environment["MYSQL_PWD"] = password; + + using var process = Process.Start(startInfo) + ?? throw new InvalidOperationException($"无法启动数据库工具 {executable}。"); + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromMinutes(options.ToolTimeoutMinutes)); + var errorTask = process.StandardError.ReadToEndAsync(timeout.Token); + var inputTask = standardInput is null + ? Task.CompletedTask + : CopyInputAsync(standardInput, process.StandardInput.BaseStream, timeout.Token); + var outputTask = standardOutput is null + ? Task.CompletedTask + : process.StandardOutput.BaseStream.CopyToAsync( + standardOutput, + timeout.Token); + + try + { + await Task.WhenAll( + process.WaitForExitAsync(timeout.Token), + inputTask, + outputTask); + } + catch (OperationCanceledException) + { + if (!process.HasExited) + process.Kill(entireProcessTree: true); + throw; + } + + var error = await errorTask; + if (process.ExitCode != 0) + throw new InvalidOperationException( + $"数据库工具执行失败（退出码 {process.ExitCode}）：{TrimToolError(error)}"); + } + + private static async Task CopyInputAsync( + Stream input, + Stream processInput, + CancellationToken cancellationToken) + { + await input.CopyToAsync(processInput, cancellationToken); + await processInput.FlushAsync(cancellationToken); + processInput.Close(); + } + + private MySqlConnectionStringBuilder GetMySqlConnectionBuilder() + { + var value = configuration.GetConnectionString("OperationsMySql"); + if (string.IsNullOrWhiteSpace(value)) + { + throw new InvalidOperationException( + "MySQL 备份与恢复演练必须配置独立的 " + + "ConnectionStrings:OperationsMySql 运维账号，不能复用日常业务账号。"); + } + var builder = new MySqlConnectionStringBuilder(value); + if (string.IsNullOrWhiteSpace(builder.Database)) + throw new InvalidOperationException( + "OperationsMySql 连接字符串未指定业务数据库名称。"); + return builder; + } + + private async Task CompleteDrillAsync( + BackupArtifact artifact, + bool succeeded, + string detail, + long durationMilliseconds, + int? tableCount, + CancellationToken cancellationToken) + { + var completedAt = DateTime.UtcNow; + var updated = artifact with + { + LastDrillAt = completedAt, + LastDrillSucceeded = succeeded, + LastDrillDetail = detail, + LastDrillDurationMilliseconds = durationMilliseconds + }; + await WriteMetadataAsync(updated, cancellationToken); + return new RestoreDrillResult( + artifact.Id, + succeeded, + completedAt, + detail, + durationMilliseconds, + tableCount); + } + + private async Task WriteMetadataAsync( + BackupArtifact artifact, + CancellationToken cancellationToken) + { + var metadataPath = Path.Combine( + backupDirectory, + $"{artifact.Id}.metadata.json"); + var temporaryPath = $"{metadataPath}.{Guid.NewGuid():N}.tmp"; + try + { + await using (var stream = new FileStream( + temporaryPath, + FileMode.CreateNew, + FileAccess.Write, + FileShare.None, + 16384, + FileOptions.Asynchronous)) + { + await JsonSerializer.SerializeAsync( + stream, + artifact, + JsonOptions, + cancellationToken); + } + File.Move(temporaryPath, metadataPath, overwrite: true); + } + finally + { + if (File.Exists(temporaryPath)) + File.Delete(temporaryPath); + } + } + + private static async Task ComputeHashAsync( + string path, + CancellationToken cancellationToken) + { + await using var stream = File.OpenRead(path); + return Convert.ToHexString( + await SHA256.HashDataAsync(stream, cancellationToken)); + } + + private static string ResolveBackupDirectory( + string configuredPath, + string contentRoot) + { + if (string.IsNullOrWhiteSpace(configuredPath)) + throw new InvalidOperationException("Operations:BackupDirectory 不能为空。"); + return Path.IsPathRooted(configuredPath) + ? Path.GetFullPath(configuredPath) + : Path.GetFullPath(configuredPath, contentRoot); + } + + private static string NormalizeProvider(string provider) => + provider.Equals("SQLite", StringComparison.OrdinalIgnoreCase) + ? "SQLite" + : provider.Equals("MySql", StringComparison.OrdinalIgnoreCase) + ? "MySql" + : throw new InvalidOperationException($"不支持数据库 Provider '{provider}'。"); + + private static string? NormalizeNote(string? note) + { + if (string.IsNullOrWhiteSpace(note)) return null; + var trimmed = note.Trim(); + return trimmed.Length <= 200 ? trimmed : trimmed[..200]; + } + + private static string SafeMessage(Exception exception) + { + var message = exception.GetBaseException().Message; + return message.Length <= 500 ? message : message[..500]; + } + + private static string TrimToolError(string error) + { + var trimmed = error.Trim(); + if (trimmed.Length == 0) return "未返回错误详情。"; + return trimmed.Length <= 500 ? trimmed : trimmed[..500]; + } +} diff --git a/src/Jiaowu.Api/Infrastructure/Operations/OperationalHealthService.cs b/src/Jiaowu.Api/Infrastructure/Operations/OperationalHealthService.cs new file mode 100644 index 0000000..cf18018 --- /dev/null +++ b/src/Jiaowu.Api/Infrastructure/Operations/OperationalHealthService.cs @@ -0,0 +1,200 @@ +using System.Diagnostics; +using Jiaowu.Api.Infrastructure.BackgroundJobs; +using Jiaowu.Api.Infrastructure.Caching; +using Jiaowu.Api.Infrastructure.Persistence; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Caching.Distributed; + +namespace Jiaowu.Api.Infrastructure.Operations; + +public sealed record OperationalComponentHealth( + string Key, + string Label, + string Status, + string Backend, + long? LatencyMilliseconds, + string Detail); + +public sealed record OperationalHealthSnapshot( + DateTime CheckedAt, + string OverallStatus, + IReadOnlyCollection Components, + BackgroundJobBacklogSnapshot? Backlog); + +public sealed class OperationalHealthService( + AppDbContext db, + IServiceProvider services, + IBackgroundJobTransport transport, + BackgroundJobMonitoringService monitoring, + DatabaseOptions databaseOptions, + AppCacheOptions cacheOptions, + IConfiguration configuration) +{ + public async Task CheckAsync( + CancellationToken cancellationToken) + { + var database = await CheckDatabaseAsync(cancellationToken); + var cache = await CheckCacheAsync(cancellationToken); + var (messaging, backlog) = await CheckMessagingAsync(cancellationToken); + var components = new[] { database, cache, messaging }; + var overall = components.Any(x => x.Status == "unhealthy") + ? "unhealthy" + : components.Any(x => x.Status == "warning") + ? "warning" + : "healthy"; + return new OperationalHealthSnapshot( + DateTime.UtcNow, + overall, + components, + backlog); + } + + private async Task CheckDatabaseAsync( + CancellationToken cancellationToken) + { + var stopwatch = Stopwatch.StartNew(); + try + { + var canConnect = await db.Database.CanConnectAsync(cancellationToken); + stopwatch.Stop(); + return canConnect + ? new OperationalComponentHealth( + "database", + "数据库", + "healthy", + databaseOptions.Provider, + stopwatch.ElapsedMilliseconds, + "连接与基础查询正常。") + : new OperationalComponentHealth( + "database", + "数据库", + "unhealthy", + databaseOptions.Provider, + stopwatch.ElapsedMilliseconds, + "无法建立数据库连接。"); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + stopwatch.Stop(); + return new OperationalComponentHealth( + "database", + "数据库", + "unhealthy", + databaseOptions.Provider, + stopwatch.ElapsedMilliseconds, + SafeMessage(exception)); + } + } + + private async Task CheckCacheAsync( + CancellationToken cancellationToken) + { + if (!cacheOptions.Enabled) + { + return new OperationalComponentHealth( + "cache", + "缓存", + "warning", + "disabled", + null, + "缓存已通过配置关闭，所有查询将直接访问数据源。"); + } + + var hasRedisConfiguration = !string.IsNullOrWhiteSpace( + configuration.GetConnectionString("Redis")); + var distributedCache = services.GetService(); + if (!hasRedisConfiguration || distributedCache is null) + { + return new OperationalComponentHealth( + "cache", + "缓存", + "healthy", + "memory", + null, + "使用进程内混合缓存；服务重启后缓存会自然重建。"); + } + + var stopwatch = Stopwatch.StartNew(); + try + { + await distributedCache.GetAsync( + "jiaowu:operations:health-probe", + cancellationToken); + stopwatch.Stop(); + return new OperationalComponentHealth( + "cache", + "缓存", + "healthy", + "redis", + stopwatch.ElapsedMilliseconds, + "Redis 连接与读取探针正常。"); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + stopwatch.Stop(); + return new OperationalComponentHealth( + "cache", + "缓存", + "unhealthy", + "redis", + stopwatch.ElapsedMilliseconds, + SafeMessage(exception)); + } + } + + private async Task<(OperationalComponentHealth Component, + BackgroundJobBacklogSnapshot? Backlog)> CheckMessagingAsync( + CancellationToken cancellationToken) + { + var stopwatch = Stopwatch.StartNew(); + BackgroundJobBacklogSnapshot? backlog = null; + try + { + var healthy = await transport.CheckHealthAsync(cancellationToken); + backlog = await monitoring.GetSnapshotAsync(cancellationToken); + stopwatch.Stop(); + var stuck = backlog.ExpiredLeases > 0 || + backlog.OldestUnfinishedAgeSeconds > 1800; + var status = !healthy + ? "unhealthy" + : stuck + ? "warning" + : "healthy"; + var detail = !healthy + ? "后台任务传输不可用。" + : stuck + ? $"发现 {backlog.ExpiredLeases} 个过期租约，最早未完成任务已等待 " + + $"{Math.Round(backlog.OldestUnfinishedAgeSeconds ?? 0)} 秒。" + : $"待发布 {backlog.Pending + backlog.Publishing}，" + + $"待处理 {backlog.Published + backlog.Processing}。"; + return ( + new OperationalComponentHealth( + "messaging", + "后台任务通道", + status, + transport.IsDurable ? "rabbitmq" : "memory", + stopwatch.ElapsedMilliseconds, + detail), + backlog); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + stopwatch.Stop(); + return ( + new OperationalComponentHealth( + "messaging", + "后台任务通道", + "unhealthy", + transport.IsDurable ? "rabbitmq" : "memory", + stopwatch.ElapsedMilliseconds, + SafeMessage(exception)), + backlog); + } + } + + private static string SafeMessage(Exception exception) + { + var message = exception.GetBaseException().Message; + return message.Length <= 300 ? message : message[..300]; + } +} diff --git a/src/Jiaowu.Api/Infrastructure/Operations/OperationsOptions.cs b/src/Jiaowu.Api/Infrastructure/Operations/OperationsOptions.cs new file mode 100644 index 0000000..7d4b15c --- /dev/null +++ b/src/Jiaowu.Api/Infrastructure/Operations/OperationsOptions.cs @@ -0,0 +1,13 @@ +namespace Jiaowu.Api.Infrastructure.Operations; + +public sealed class OperationsOptions +{ + public const string SectionName = "Operations"; + + public string BackupDirectory { get; set; } = "data/backups"; + public int BackupWarningHours { get; set; } = 24; + public int ToolTimeoutMinutes { get; set; } = 30; + public string MySqlDumpPath { get; set; } = "mysqldump"; + public string MySqlClientPath { get; set; } = "mysql"; + public string[] MySqlAdditionalArguments { get; set; } = []; +} diff --git a/src/Jiaowu.Api/Program.cs b/src/Jiaowu.Api/Program.cs index b21e8a1..01ca704 100644 --- a/src/Jiaowu.Api/Program.cs +++ b/src/Jiaowu.Api/Program.cs @@ -8,6 +8,7 @@ using Jiaowu.Api.Infrastructure.Caching; using Jiaowu.Api.Infrastructure.Exams; using Jiaowu.Api.Infrastructure.Middleware; using Jiaowu.Api.Infrastructure.OfficialDocuments; +using Jiaowu.Api.Infrastructure.Operations; using Jiaowu.Api.Infrastructure.Persistence; using Jiaowu.Api.Infrastructure.Scheduling; using Jiaowu.Api.Infrastructure.Timetables; @@ -74,6 +75,9 @@ var officialDocumentOptions = builder.Configuration var backgroundJobOptions = builder.Configuration .GetSection(BackgroundJobOptions.SectionName) .Get() ?? new BackgroundJobOptions(); +var operationsOptions = builder.Configuration + .GetSection(OperationsOptions.SectionName) + .Get() ?? new OperationsOptions(); var rabbitMqOptions = builder.Configuration .GetSection(RabbitMqOptions.SectionName) .Get() ?? new RabbitMqOptions(); @@ -167,10 +171,25 @@ if (backgroundJobOptions.UsesRabbitMq && "生产环境启用 RabbitMQ 时不能使用默认 guest 凭据。"); } +if (string.IsNullOrWhiteSpace(operationsOptions.BackupDirectory) || + operationsOptions.BackupWarningHours is < 1 or > 8760 || + operationsOptions.ToolTimeoutMinutes is < 1 or > 240 || + string.IsNullOrWhiteSpace(operationsOptions.MySqlDumpPath) || + string.IsNullOrWhiteSpace(operationsOptions.MySqlClientPath) || + operationsOptions.MySqlAdditionalArguments.Length > 20 || + operationsOptions.MySqlAdditionalArguments.Any(argument => + string.IsNullOrWhiteSpace(argument) || + argument.Length > 300 || + !argument.StartsWith("--", StringComparison.Ordinal))) +{ + throw new InvalidOperationException("Operations 运维与备份配置超出允许范围。"); +} + builder.Services.AddSingleton(databaseOptions); builder.Services.AddSingleton(cacheOptions); builder.Services.AddSingleton(officialDocumentOptions); builder.Services.AddSingleton(backgroundJobOptions); +builder.Services.AddSingleton(operationsOptions); builder.Services.AddSingleton(rabbitMqOptions); builder.Services.Configure( builder.Configuration.GetSection(OfficialDocumentOptions.SectionName)); @@ -277,6 +296,8 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddSingleton(); builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); if (backgroundJobOptions.UsesRabbitMq) { diff --git a/src/Jiaowu.Api/appsettings.json b/src/Jiaowu.Api/appsettings.json index a07516c..7c4fffe 100644 --- a/src/Jiaowu.Api/appsettings.json +++ b/src/Jiaowu.Api/appsettings.json @@ -19,6 +19,14 @@ "AnalyticsLocalExpirationSeconds": 30, "MaximumPayloadKilobytes": 2048 }, + "Operations": { + "BackupDirectory": "data/backups", + "BackupWarningHours": 24, + "ToolTimeoutMinutes": 30, + "MySqlDumpPath": "mysqldump", + "MySqlClientPath": "mysql", + "MySqlAdditionalArguments": [] + }, "BackgroundJobs": { "Transport": "InMemory", "PollIntervalMilliseconds": 500, diff --git a/tests/Jiaowu.Api.Tests/OperationsControllerTests.cs b/tests/Jiaowu.Api.Tests/OperationsControllerTests.cs new file mode 100644 index 0000000..d7dabfd --- /dev/null +++ b/tests/Jiaowu.Api.Tests/OperationsControllerTests.cs @@ -0,0 +1,252 @@ +using System.Reflection; +using Jiaowu.Api.Contracts; +using Jiaowu.Api.Controllers; +using Jiaowu.Api.Domain.Academic; +using Jiaowu.Api.Domain.Identity; +using Jiaowu.Api.Domain.System; +using Jiaowu.Api.Infrastructure.BackgroundJobs; +using Jiaowu.Api.Infrastructure.Caching; +using Jiaowu.Api.Infrastructure.Operations; +using Jiaowu.Api.Infrastructure.Persistence; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.FileProviders; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Jiaowu.Api.Tests; + +public sealed class OperationsControllerTests +{ + [Fact] + public void Controller_is_restricted_to_super_admin() + { + var authorize = typeof(OperationsController) + .GetCustomAttribute(); + + Assert.NotNull(authorize); + Assert.Equal(SystemRoles.SuperAdmin, authorize.Roles); + } + + [Fact] + public async Task Audit_and_failed_job_queries_return_operational_records() + { + var root = CreateTemporaryRoot(); + try + { + await using var fixture = await OperationsFixture.CreateAsync(root); + var term = new AcademicTerm + { + Code = "OPS-TERM", + Name = "运维测试学期", + AcademicYear = "2026-2027", + Season = TermSeason.Autumn, + StartDate = new DateOnly(2026, 9, 1), + EndDate = new DateOnly(2027, 1, 15) + }; + var plan = new SchedulePlan + { + AcademicTerm = term, + Name = "恢复演练排课方案", + Version = "V1" + }; + var job = new AutomaticScheduleJob + { + SchedulePlan = plan, + Status = AutomaticScheduleJobStatus.Failed, + ErrorMessage = "排课求解器未找到可行解。", + CompletedAt = DateTime.UtcNow + }; + var outbox = BackgroundJobOutboxMessage.Create( + BackgroundJobKind.AutomaticSchedule, + job.Id); + outbox.State = BackgroundJobOutboxState.Completed; + outbox.ProcessingAttempts = 3; + fixture.Db.AuditLogs.Add(new AuditLog + { + UserName = "root", + Method = "POST", + Path = "/api/schedules", + StatusCode = 500, + IpAddress = "127.0.0.1" + }); + fixture.Db.AddRange(term, plan, job, outbox); + await fixture.Db.SaveChangesAsync(); + + var auditAction = await fixture.Controller.GetAuditLogs( + cancellationToken: CancellationToken.None); + var auditResult = Assert.IsType(auditAction.Result); + var auditPage = Assert.IsType>(auditResult.Value); + Assert.Single(auditPage.Items); + Assert.Equal(500, auditPage.Items.Single().StatusCode); + + var jobsAction = await fixture.Controller.GetFailedJobs( + cancellationToken: CancellationToken.None); + var jobsResult = Assert.IsType(jobsAction.Result); + var jobsPage = + Assert.IsType>(jobsResult.Value); + var failed = Assert.Single(jobsPage.Items); + Assert.Equal("AutomaticSchedule", failed.Kind); + Assert.Equal(3, failed.ProcessingAttempts); + Assert.Contains("可行解", failed.ErrorMessage); + } + finally + { + DeleteTemporaryRoot(root); + } + } + + [Fact] + public async Task Sqlite_backup_can_be_verified_in_an_isolated_restore_drill() + { + var root = CreateTemporaryRoot(); + try + { + await using var fixture = await OperationsFixture.CreateAsync(root); + fixture.Db.AuditLogs.Add(new AuditLog + { + UserName = "backup-test", + Method = "POST", + Path = "/api/test", + StatusCode = 204 + }); + await fixture.Db.SaveChangesAsync(); + + var artifact = await fixture.Backups.CreateAsync( + "自动化测试", + CancellationToken.None); + var result = await fixture.Backups.RunRestoreDrillAsync( + artifact.Id, + CancellationToken.None); + var listed = await fixture.Backups.ListAsync(CancellationToken.None); + + Assert.True(File.Exists(Path.Combine( + root, + "backups", + artifact.FileName))); + Assert.Equal(64, artifact.Sha256.Length); + Assert.True(result.Succeeded, result.Detail); + Assert.True(result.TableCount > 0); + Assert.True(Assert.Single(listed).LastDrillSucceeded); + } + finally + { + DeleteTemporaryRoot(root); + } + } + + private static string CreateTemporaryRoot() + { + var root = Path.Combine( + Path.GetTempPath(), + $"jiaowu-operations-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + return root; + } + + private static void DeleteTemporaryRoot(string root) + { + var resolved = Path.GetFullPath(root); + var temporary = Path.GetFullPath(Path.GetTempPath()); + Assert.StartsWith(temporary, resolved, StringComparison.OrdinalIgnoreCase); + if (Directory.Exists(resolved)) + Directory.Delete(resolved, recursive: true); + } + + private sealed class OperationsFixture : IAsyncDisposable + { + private readonly ServiceProvider provider; + + private OperationsFixture( + ServiceProvider provider, + AppDbContext db, + OperationsController controller, + DatabaseBackupService backups) + { + this.provider = provider; + Db = db; + Controller = controller; + Backups = backups; + } + + public AppDbContext Db { get; } + public OperationsController Controller { get; } + public DatabaseBackupService Backups { get; } + + public static async Task CreateAsync(string root) + { + var databasePath = Path.Combine(root, "source.sqlite"); + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["ConnectionStrings:SQLite"] = + $"Data Source={databasePath};Pooling=False" + }) + .Build(); + var databaseOptions = new DatabaseOptions { Provider = "SQLite" }; + var cacheOptions = new AppCacheOptions(); + var operationsOptions = new OperationsOptions + { + BackupDirectory = "backups" + }; + var dbContextOptions = new DbContextOptionsBuilder() + .UseSqlite(configuration.GetConnectionString("SQLite")) + .Options; + var db = new AppDbContext(dbContextOptions); + await db.Database.EnsureCreatedAsync(); + + var services = new ServiceCollection() + .AddLogging() + .BuildServiceProvider(); + var logger = services.GetRequiredService< + ILogger>(); + var environment = new TestHostEnvironment + { + ContentRootPath = root + }; + var transport = new InMemoryBackgroundJobTransport(); + var monitoring = new BackgroundJobMonitoringService(db); + var health = new OperationalHealthService( + db, + services, + transport, + monitoring, + databaseOptions, + cacheOptions, + configuration); + var backups = new DatabaseBackupService( + databaseOptions, + operationsOptions, + configuration, + environment, + logger); + var controller = new OperationsController( + db, + health, + backups, + operationsOptions); + return new OperationsFixture(services, db, controller, backups); + } + + public async ValueTask DisposeAsync() + { + await Db.DisposeAsync(); + await provider.DisposeAsync(); + } + } + + private sealed class TestHostEnvironment : IWebHostEnvironment + { + public string ApplicationName { get; set; } = "Jiaowu.Api.Tests"; + public IFileProvider WebRootFileProvider { get; set; } = new NullFileProvider(); + public string WebRootPath { get; set; } = ""; + public string EnvironmentName { get; set; } = Environments.Development; + public string ContentRootPath { get; set; } = ""; + public IFileProvider ContentRootFileProvider { get; set; } = + new NullFileProvider(); + } +} diff --git a/web/src/layouts/AdminLayout.vue b/web/src/layouts/AdminLayout.vue index 527dd9c..e31e090 100644 --- a/web/src/layouts/AdminLayout.vue +++ b/web/src/layouts/AdminLayout.vue @@ -97,6 +97,7 @@ const navigationGroups = computed(() => [ { path: '/students', label: '学生档案' }, ), ...whenVisible(auth.isSuperAdmin, { path: '/users', label: '用户与权限' }), + ...whenVisible(auth.isSuperAdmin, { path: '/operations', label: '运维与审计' }), ], }, { diff --git a/web/src/router/index.ts b/web/src/router/index.ts index 850a744..eec0bc8 100644 --- a/web/src/router/index.ts +++ b/web/src/router/index.ts @@ -291,6 +291,12 @@ const router = createRouter({ component: () => import('../views/UsersView.vue'), meta: { roles: ['SuperAdmin'] }, }, + { + path: 'operations', + name: 'operations', + component: () => import('../views/OperationsConsoleView.vue'), + meta: { roles: ['SuperAdmin'] }, + }, { path: 'evaluations', name: 'evaluations', diff --git a/web/src/views/OperationsConsoleView.vue b/web/src/views/OperationsConsoleView.vue new file mode 100644 index 0000000..42648dd --- /dev/null +++ b/web/src/views/OperationsConsoleView.vue @@ -0,0 +1,1027 @@ + + + + +