“运维与审计控制台”。
主要能力: SuperAdmin 专用入口:组织与权限 → 运维与审计。 操作日志分页查询,支持时间、账号、路径、方法和状态码筛选。 汇总自动排课、课表发布、补考安排三类失败后台任务。 实时检查数据库、缓存、任务通道及积压状态。 聚合 5xx、失败/重试任务、健康探针和备份时效告警。 SQLite 在线备份;MySQL 调用原生客户端备份。 SHA-256 校验及隔离数据库恢复演练,不覆盖业务库。 MySQL 强制使用独立运维连接,容器增加持久化备份卷与数据库客户端。
This commit is contained in:
@@ -6,6 +6,9 @@ Database__Provider=MySql
|
|||||||
Database__ApplyMigrationsOnStartup=false
|
Database__ApplyMigrationsOnStartup=false
|
||||||
Database__CommandTimeoutSeconds=30
|
Database__CommandTimeoutSeconds=30
|
||||||
ConnectionStrings__MySql="Server=db.example.edu.cn;Port=3306;Database=jiaowu;User=APP_USER;Password=REPLACE_WITH_A_STRONG_PASSWORD;SslMode=VerifyFull;SslCa=/etc/jiaowu/mysql-ca.pem;"
|
ConnectionStrings__MySql="Server=db.example.edu.cn;Port=3306;Database=jiaowu;User=APP_USER;Password=REPLACE_WITH_A_STRONG_PASSWORD;SslMode=VerifyFull;SslCa=/etc/jiaowu/mysql-ca.pem;"
|
||||||
|
# 仅供备份与隔离恢复演练使用。该账号需要读取业务库,并仅能创建/删除
|
||||||
|
# jiaowu_restore_drill_* 临时库;不要在此复用日常业务账号。
|
||||||
|
# ConnectionStrings__OperationsMySql="Server=db.example.edu.cn;Port=3306;Database=jiaowu;User=OPS_USER;Password=REPLACE_WITH_A_STRONG_PASSWORD;SslMode=VerifyFull;SslCa=/etc/jiaowu/mysql-ca.pem;"
|
||||||
# Redis 是可选加速器;留空时应用仅使用进程内缓存。
|
# Redis 是可选加速器;留空时应用仅使用进程内缓存。
|
||||||
# ConnectionStrings__Redis="redis.example.edu.cn:6380,user=jiaowu,password=REPLACE_WITH_A_STRONG_PASSWORD,ssl=true,abortConnect=false"
|
# ConnectionStrings__Redis="redis.example.edu.cn:6380,user=jiaowu,password=REPLACE_WITH_A_STRONG_PASSWORD,ssl=true,abortConnect=false"
|
||||||
|
|
||||||
@@ -32,6 +35,16 @@ Cache__AnalyticsExpirationMinutes=3
|
|||||||
Cache__AnalyticsLocalExpirationSeconds=30
|
Cache__AnalyticsLocalExpirationSeconds=30
|
||||||
Cache__MaximumPayloadKilobytes=2048
|
Cache__MaximumPayloadKilobytes=2048
|
||||||
|
|
||||||
|
# 运维控制台备份目录必须位于持久化、仅服务账号可写的位置。
|
||||||
|
Operations__BackupDirectory=/var/lib/jiaowu/backups
|
||||||
|
Operations__BackupWarningHours=24
|
||||||
|
Operations__ToolTimeoutMinutes=30
|
||||||
|
Operations__MySqlDumpPath=mysqldump
|
||||||
|
Operations__MySqlClientPath=mysql
|
||||||
|
# 按实际客户端补充 TLS 参数;Oracle MySQL 客户端示例:
|
||||||
|
# Operations__MySqlAdditionalArguments__0=--ssl-mode=VERIFY_IDENTITY
|
||||||
|
# Operations__MySqlAdditionalArguments__1=--ssl-ca=/etc/jiaowu/mysql-ca.pem
|
||||||
|
|
||||||
Jwt__Issuer=Jiaowu.Api
|
Jwt__Issuer=Jiaowu.Api
|
||||||
Jwt__Audience=Jiaowu.Web
|
Jwt__Audience=Jiaowu.Web
|
||||||
Jwt__Key=REPLACE_WITH_AT_LEAST_32_RANDOM_BYTES
|
Jwt__Key=REPLACE_WITH_AT_LEAST_32_RANDOM_BYTES
|
||||||
|
|||||||
+9
-2
@@ -30,11 +30,14 @@ FROM mcr.microsoft.com/dotnet/aspnet:10.0-alpine AS final
|
|||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
ENV ASPNETCORE_ENVIRONMENT=Production \
|
ENV ASPNETCORE_ENVIRONMENT=Production \
|
||||||
ASPNETCORE_HTTP_PORTS=8080 \
|
ASPNETCORE_HTTP_PORTS=8080 \
|
||||||
DOTNET_EnableDiagnostics=0
|
DOTNET_EnableDiagnostics=0 \
|
||||||
|
Operations__BackupDirectory=/var/lib/jiaowu/backups \
|
||||||
|
Operations__MySqlDumpPath=mariadb-dump \
|
||||||
|
Operations__MySqlClientPath=mariadb
|
||||||
EXPOSE 8080
|
EXPOSE 8080
|
||||||
COPY --from=build /app/publish/ ./
|
COPY --from=build /app/publish/ ./
|
||||||
ARG UID=10001
|
ARG UID=10001
|
||||||
RUN apk add --no-cache font-noto-cjk
|
RUN apk add --no-cache font-noto-cjk mariadb-client
|
||||||
RUN adduser \
|
RUN adduser \
|
||||||
--disabled-password \
|
--disabled-password \
|
||||||
--gecos "" \
|
--gecos "" \
|
||||||
@@ -43,5 +46,9 @@ RUN adduser \
|
|||||||
--no-create-home \
|
--no-create-home \
|
||||||
--uid "${UID}" \
|
--uid "${UID}" \
|
||||||
appuser
|
appuser
|
||||||
|
RUN mkdir -p /var/lib/jiaowu/backups \
|
||||||
|
&& chown appuser:appuser /var/lib/jiaowu/backups \
|
||||||
|
&& chmod 700 /var/lib/jiaowu/backups
|
||||||
|
VOLUME ["/var/lib/jiaowu/backups"]
|
||||||
USER appuser
|
USER appuser
|
||||||
ENTRYPOINT ["dotnet", "Jiaowu.Api.dll"]
|
ENTRYPOINT ["dotnet", "Jiaowu.Api.dll"]
|
||||||
|
|||||||
@@ -101,6 +101,12 @@ sudo chown -R root:jiaowu /opt/jiaowu
|
|||||||
sudo chmod 0750 /opt/jiaowu
|
sudo chmod 0750 /opt/jiaowu
|
||||||
sudo chmod 0750 /opt/jiaowu/Jiaowu.Api
|
sudo chmod 0750 /opt/jiaowu/Jiaowu.Api
|
||||||
sudo chmod 0640 /opt/jiaowu/.env
|
sudo chmod 0640 /opt/jiaowu/.env
|
||||||
|
sudo install -d \
|
||||||
|
--owner=jiaowu \
|
||||||
|
--group=jiaowu \
|
||||||
|
--mode=0700 \
|
||||||
|
/var/lib/jiaowu/backups
|
||||||
|
sudo apt-get install default-mysql-client
|
||||||
```
|
```
|
||||||
|
|
||||||
如果账号已存在,`useradd` 会报错,可以跳过该命令。RHEL 系发行版的 `nologin` 通常
|
如果账号已存在,`useradd` 会报错,可以跳过该命令。RHEL 系发行版的 `nologin` 通常
|
||||||
@@ -297,6 +303,25 @@ Outbox 租约恢复改为按维护周期执行,避免积压发布时每条消
|
|||||||
消息会根据 Outbox 状态和租约继续补投。迁移服务应先应用
|
消息会根据 Outbox 状态和租约继续补投。迁移服务应先应用
|
||||||
`BackgroundJobOutbox` 数据库迁移,再启动应用实例。
|
`BackgroundJobOutbox` 数据库迁移,再启动应用实例。
|
||||||
|
|
||||||
|
### 运维与审计控制台
|
||||||
|
|
||||||
|
超级管理员可从“组织与权限 → 运维与审计”查询写操作日志、三类失败后台任务、数据库、
|
||||||
|
缓存与任务通道健康状态,并查看由 5xx、失败/重试任务、健康探针和备份时效汇总出的异常
|
||||||
|
告警。查询接口和备份操作均在后端强制要求 `SuperAdmin`,不能只依赖前端菜单隐藏。
|
||||||
|
|
||||||
|
SQLite 开发环境直接使用在线备份 API。MySQL 环境需要在服务器安装 `mysqldump` 与
|
||||||
|
`mysql`(容器镜像已包含对应的 `mariadb-dump` 与 `mariadb` 客户端),并配置独立的
|
||||||
|
`ConnectionStrings__OperationsMySql`。该账号不得复用日常业务账号:它需要读取业务库,
|
||||||
|
并只应被授权创建和删除名称为 `jiaowu_restore_drill_*` 的临时演练库。恢复演练不会覆盖
|
||||||
|
当前业务库,流程是“校验 SHA-256 → 恢复到随机临时库 → 检查表结构 → 删除临时库”。
|
||||||
|
|
||||||
|
备份目录必须是仅服务账号可写的持久化目录。示例配置使用
|
||||||
|
`/var/lib/jiaowu/backups`;Compose 已挂载独立命名卷。启用 MySQL TLS 时,还要通过
|
||||||
|
`Operations__MySqlAdditionalArguments__N` 传入与所选命令行客户端匹配的 CA 与主机名
|
||||||
|
校验参数。例如 Oracle MySQL 客户端使用 `--ssl-mode=VERIFY_IDENTITY` 和
|
||||||
|
`--ssl-ca=/etc/jiaowu/mysql-ca.pem`,容器内 MariaDB 客户端使用 `--ssl`、
|
||||||
|
`--ssl-ca=...` 与 `--ssl-verify-server-cert`。
|
||||||
|
|
||||||
## 跨平台发布与 Docker
|
## 跨平台发布与 Docker
|
||||||
|
|
||||||
`.gitea/workflows/publish.yml` 只在推送 `v*` 标签或手动运行时执行,普通分支 push
|
`.gitea/workflows/publish.yml` 只在推送 `v*` 标签或手动运行时执行,普通分支 push
|
||||||
|
|||||||
@@ -17,11 +17,15 @@ services:
|
|||||||
options:
|
options:
|
||||||
max-size: "10m"
|
max-size: "10m"
|
||||||
max-file: "3"
|
max-file: "3"
|
||||||
|
volumes:
|
||||||
|
- jiaowu-backups:/var/lib/jiaowu/backups
|
||||||
|
|
||||||
# 如果 .env 中的 SslCa=/etc/jiaowu/mysql-ca.pem,请把 CA 放到
|
# 如果 .env 中的 SslCa=/etc/jiaowu/mysql-ca.pem,请把 CA 放到
|
||||||
# ./certs/mysql-ca.pem,并取消下面三行注释。
|
# ./certs/mysql-ca.pem,并在上面的 volumes 中追加以下四行。
|
||||||
# volumes:
|
|
||||||
# - type: bind
|
# - type: bind
|
||||||
# source: ./certs/mysql-ca.pem
|
# source: ./certs/mysql-ca.pem
|
||||||
# target: /etc/jiaowu/mysql-ca.pem
|
# target: /etc/jiaowu/mysql-ca.pem
|
||||||
# read_only: true
|
# read_only: true
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
jiaowu-backups:
|
||||||
|
|||||||
@@ -138,6 +138,8 @@ services:
|
|||||||
- "${JIAOWU_PORT:-8080}:8080"
|
- "${JIAOWU_PORT:-8080}:8080"
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
init: true
|
init: true
|
||||||
|
volumes:
|
||||||
|
- backup-data:/var/lib/jiaowu/backups
|
||||||
logging: *json-logging
|
logging: *json-logging
|
||||||
|
|
||||||
# 工具型一次性服务:普通 docker compose up 不会执行它。
|
# 工具型一次性服务:普通 docker compose up 不会执行它。
|
||||||
@@ -161,3 +163,4 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
mysql-data:
|
mysql-data:
|
||||||
rabbitmq-data:
|
rabbitmq-data:
|
||||||
|
backup-data:
|
||||||
|
|||||||
@@ -0,0 +1,519 @@
|
|||||||
|
using System.ComponentModel.DataAnnotations;
|
||||||
|
using Jiaowu.Api.Contracts;
|
||||||
|
using Jiaowu.Api.Domain.Academic;
|
||||||
|
using Jiaowu.Api.Domain.Identity;
|
||||||
|
using Jiaowu.Api.Domain.System;
|
||||||
|
using Jiaowu.Api.Infrastructure.Operations;
|
||||||
|
using Jiaowu.Api.Infrastructure.Persistence;
|
||||||
|
using Microsoft.AspNetCore.Authorization;
|
||||||
|
using Microsoft.AspNetCore.Mvc;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
|
||||||
|
namespace Jiaowu.Api.Controllers;
|
||||||
|
|
||||||
|
[ApiController]
|
||||||
|
[Authorize(Roles = SystemRoles.SuperAdmin)]
|
||||||
|
[Route("api/operations")]
|
||||||
|
public sealed class OperationsController(
|
||||||
|
AppDbContext db,
|
||||||
|
OperationalHealthService healthService,
|
||||||
|
DatabaseBackupService backupService,
|
||||||
|
OperationsOptions options) : ControllerBase
|
||||||
|
{
|
||||||
|
[HttpGet("summary")]
|
||||||
|
public async Task<ActionResult<OperationsSummary>> GetSummary(
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var since = DateTime.UtcNow.AddHours(-24);
|
||||||
|
var health = await healthService.CheckAsync(cancellationToken);
|
||||||
|
var backups = await backupService.ListAsync(cancellationToken);
|
||||||
|
var auditCount = await db.AuditLogs.AsNoTracking()
|
||||||
|
.CountAsync(x => x.CreatedAt >= since, cancellationToken);
|
||||||
|
var serverErrorCount = await db.AuditLogs.AsNoTracking()
|
||||||
|
.CountAsync(
|
||||||
|
x => x.CreatedAt >= since && x.StatusCode >= 500,
|
||||||
|
cancellationToken);
|
||||||
|
var failedJobCount = await CountFailedJobsAsync(
|
||||||
|
DateTime.UtcNow.AddDays(-7),
|
||||||
|
cancellationToken);
|
||||||
|
var alerts = await BuildAlertsAsync(
|
||||||
|
health,
|
||||||
|
backups,
|
||||||
|
serverErrorCount,
|
||||||
|
failedJobCount,
|
||||||
|
cancellationToken);
|
||||||
|
return Ok(new OperationsSummary(
|
||||||
|
DateTime.UtcNow,
|
||||||
|
health,
|
||||||
|
new OperationsCounters(
|
||||||
|
auditCount,
|
||||||
|
serverErrorCount,
|
||||||
|
failedJobCount,
|
||||||
|
alerts.Count(x => x.Severity == "critical")),
|
||||||
|
alerts,
|
||||||
|
backups.FirstOrDefault()));
|
||||||
|
}
|
||||||
|
|
||||||
|
[HttpGet("health")]
|
||||||
|
public async Task<ActionResult<OperationalHealthSnapshot>> GetHealth(
|
||||||
|
CancellationToken cancellationToken) =>
|
||||||
|
Ok(await healthService.CheckAsync(cancellationToken));
|
||||||
|
|
||||||
|
[HttpGet("audit-logs")]
|
||||||
|
public async Task<ActionResult<PagedResult<AuditLogItem>>> GetAuditLogs(
|
||||||
|
[FromQuery] int page = 1,
|
||||||
|
[FromQuery] int pageSize = 20,
|
||||||
|
[FromQuery] string? method = null,
|
||||||
|
[FromQuery] int? statusCode = null,
|
||||||
|
[FromQuery] string? userName = null,
|
||||||
|
[FromQuery] string? path = null,
|
||||||
|
[FromQuery] DateTime? from = null,
|
||||||
|
[FromQuery] DateTime? to = null,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
var pagingError = ValidatePaging(page, pageSize);
|
||||||
|
if (pagingError is not null) return pagingError;
|
||||||
|
var rangeError = ValidateRange(from, to);
|
||||||
|
if (rangeError is not null) return rangeError;
|
||||||
|
|
||||||
|
var query = db.AuditLogs.AsNoTracking().AsQueryable();
|
||||||
|
if (!string.IsNullOrWhiteSpace(method))
|
||||||
|
{
|
||||||
|
var normalizedMethod = method.Trim().ToUpperInvariant();
|
||||||
|
query = query.Where(x => x.Method == normalizedMethod);
|
||||||
|
}
|
||||||
|
if (statusCode.HasValue)
|
||||||
|
query = query.Where(x => x.StatusCode == statusCode.Value);
|
||||||
|
if (!string.IsNullOrWhiteSpace(userName))
|
||||||
|
{
|
||||||
|
var normalizedUser = userName.Trim();
|
||||||
|
query = query.Where(x =>
|
||||||
|
x.UserName != null && x.UserName.Contains(normalizedUser));
|
||||||
|
}
|
||||||
|
if (!string.IsNullOrWhiteSpace(path))
|
||||||
|
{
|
||||||
|
var normalizedPath = path.Trim();
|
||||||
|
query = query.Where(x => x.Path.Contains(normalizedPath));
|
||||||
|
}
|
||||||
|
query = query.Where(x =>
|
||||||
|
x.CreatedAt >= (from ?? DateTime.UtcNow.AddDays(-1)));
|
||||||
|
if (to.HasValue)
|
||||||
|
query = query.Where(x => x.CreatedAt <= to.Value);
|
||||||
|
|
||||||
|
var total = await query.CountAsync(cancellationToken);
|
||||||
|
var items = await query
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.Skip((page - 1) * pageSize)
|
||||||
|
.Take(pageSize)
|
||||||
|
.Select(x => new AuditLogItem(
|
||||||
|
x.Id,
|
||||||
|
x.UserName,
|
||||||
|
x.Method,
|
||||||
|
x.Path,
|
||||||
|
x.StatusCode,
|
||||||
|
x.IpAddress,
|
||||||
|
x.CreatedAt))
|
||||||
|
.ToListAsync(cancellationToken);
|
||||||
|
return Ok(new PagedResult<AuditLogItem>(items, total, page, pageSize));
|
||||||
|
}
|
||||||
|
|
||||||
|
[HttpGet("failed-jobs")]
|
||||||
|
public async Task<ActionResult<PagedResult<FailedBackgroundJobItem>>>
|
||||||
|
GetFailedJobs(
|
||||||
|
[FromQuery] int page = 1,
|
||||||
|
[FromQuery] int pageSize = 20,
|
||||||
|
[FromQuery] string? kind = null,
|
||||||
|
[FromQuery] DateTime? from = null,
|
||||||
|
[FromQuery] DateTime? to = null,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
var pagingError = ValidatePaging(page, pageSize);
|
||||||
|
if (pagingError is not null) return pagingError;
|
||||||
|
var rangeError = ValidateRange(from, to);
|
||||||
|
if (rangeError is not null) return rangeError;
|
||||||
|
|
||||||
|
var normalizedKind = NormalizeJobKind(kind);
|
||||||
|
if (kind is not null && normalizedKind is null)
|
||||||
|
return ValidationProblem("后台任务类型无效。");
|
||||||
|
|
||||||
|
var effectiveFrom = from ?? DateTime.UtcNow.AddDays(-30);
|
||||||
|
var take = checked(page * pageSize);
|
||||||
|
var rows = new List<FailedBackgroundJobItem>();
|
||||||
|
var total = 0;
|
||||||
|
|
||||||
|
if (normalizedKind is null or "AutomaticSchedule")
|
||||||
|
{
|
||||||
|
var query = db.AutomaticScheduleJobs.AsNoTracking()
|
||||||
|
.Where(x =>
|
||||||
|
x.Status == AutomaticScheduleJobStatus.Failed &&
|
||||||
|
x.CreatedAt >= effectiveFrom);
|
||||||
|
if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value);
|
||||||
|
total += await query.CountAsync(cancellationToken);
|
||||||
|
rows.AddRange(await query
|
||||||
|
.OrderByDescending(x => x.CompletedAt ?? x.UpdatedAt)
|
||||||
|
.Take(take)
|
||||||
|
.Select(x => new FailedBackgroundJobItem(
|
||||||
|
x.Id,
|
||||||
|
"AutomaticSchedule",
|
||||||
|
"自动排课",
|
||||||
|
x.SchedulePlan == null ? "排课方案" : x.SchedulePlan.Name,
|
||||||
|
x.ErrorMessage ?? "任务失败但未记录错误详情。",
|
||||||
|
x.CreatedAt,
|
||||||
|
x.StartedAt,
|
||||||
|
x.CompletedAt,
|
||||||
|
0))
|
||||||
|
.ToListAsync(cancellationToken));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalizedKind is null or "SchedulePublish")
|
||||||
|
{
|
||||||
|
var query = db.SchedulePublishJobs.AsNoTracking()
|
||||||
|
.Where(x =>
|
||||||
|
x.Status == SchedulePublishJobStatus.Failed &&
|
||||||
|
x.CreatedAt >= effectiveFrom);
|
||||||
|
if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value);
|
||||||
|
total += await query.CountAsync(cancellationToken);
|
||||||
|
rows.AddRange(await query
|
||||||
|
.OrderByDescending(x => x.CompletedAt ?? x.UpdatedAt)
|
||||||
|
.Take(take)
|
||||||
|
.Select(x => new FailedBackgroundJobItem(
|
||||||
|
x.Id,
|
||||||
|
"SchedulePublish",
|
||||||
|
"课表发布",
|
||||||
|
x.SchedulePlan == null ? "排课方案" : x.SchedulePlan.Name,
|
||||||
|
x.ErrorMessage ?? "任务失败但未记录错误详情。",
|
||||||
|
x.CreatedAt,
|
||||||
|
x.StartedAt,
|
||||||
|
x.CompletedAt,
|
||||||
|
0))
|
||||||
|
.ToListAsync(cancellationToken));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalizedKind is null or "MakeupExamAuto")
|
||||||
|
{
|
||||||
|
var query = db.MakeupExamAutoJobs.AsNoTracking()
|
||||||
|
.Where(x =>
|
||||||
|
x.Status == MakeupExamAutoJobStatus.Failed &&
|
||||||
|
x.CreatedAt >= effectiveFrom);
|
||||||
|
if (to.HasValue) query = query.Where(x => x.CreatedAt <= to.Value);
|
||||||
|
total += await query.CountAsync(cancellationToken);
|
||||||
|
rows.AddRange(await query
|
||||||
|
.OrderByDescending(x => x.CompletedAt ?? x.UpdatedAt)
|
||||||
|
.Take(take)
|
||||||
|
.Select(x => new FailedBackgroundJobItem(
|
||||||
|
x.Id,
|
||||||
|
"MakeupExamAuto",
|
||||||
|
"补考自动安排",
|
||||||
|
x.MakeupExamPlan == null ? "补考计划" : x.MakeupExamPlan.Name,
|
||||||
|
x.ErrorMessage ?? "任务失败但未记录错误详情。",
|
||||||
|
x.CreatedAt,
|
||||||
|
x.StartedAt,
|
||||||
|
x.CompletedAt,
|
||||||
|
0))
|
||||||
|
.ToListAsync(cancellationToken));
|
||||||
|
}
|
||||||
|
|
||||||
|
var pageItems = rows
|
||||||
|
.OrderByDescending(x => x.CompletedAt ?? x.CreatedAt)
|
||||||
|
.Skip((page - 1) * pageSize)
|
||||||
|
.Take(pageSize)
|
||||||
|
.ToArray();
|
||||||
|
if (pageItems.Length > 0)
|
||||||
|
{
|
||||||
|
var ids = pageItems.Select(x => x.Id).ToArray();
|
||||||
|
var attempts = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||||
|
.Where(x => ids.Contains(x.JobId))
|
||||||
|
.Select(x => new { x.JobId, x.ProcessingAttempts })
|
||||||
|
.ToDictionaryAsync(x => x.JobId, x => x.ProcessingAttempts,
|
||||||
|
cancellationToken);
|
||||||
|
pageItems = pageItems
|
||||||
|
.Select(x => x with
|
||||||
|
{
|
||||||
|
ProcessingAttempts = attempts.GetValueOrDefault(x.Id)
|
||||||
|
})
|
||||||
|
.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
return Ok(new PagedResult<FailedBackgroundJobItem>(
|
||||||
|
pageItems,
|
||||||
|
total,
|
||||||
|
page,
|
||||||
|
pageSize));
|
||||||
|
}
|
||||||
|
|
||||||
|
[HttpGet("backups")]
|
||||||
|
public async Task<ActionResult<IReadOnlyCollection<BackupArtifact>>> GetBackups(
|
||||||
|
CancellationToken cancellationToken) =>
|
||||||
|
Ok(await backupService.ListAsync(cancellationToken));
|
||||||
|
|
||||||
|
[HttpPost("backups")]
|
||||||
|
public async Task<ActionResult<BackupArtifact>> CreateBackup(
|
||||||
|
CreateBackupRequest request,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var artifact = await backupService.CreateAsync(
|
||||||
|
request.Note,
|
||||||
|
cancellationToken);
|
||||||
|
return CreatedAtAction(
|
||||||
|
nameof(GetBackups),
|
||||||
|
new { id = artifact.Id },
|
||||||
|
artifact);
|
||||||
|
}
|
||||||
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||||
|
{
|
||||||
|
return Problem(
|
||||||
|
title: "数据库备份失败",
|
||||||
|
detail: SafeMessage(exception),
|
||||||
|
statusCode: StatusCodes.Status503ServiceUnavailable);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[HttpPost("backups/{backupId}/restore-drill")]
|
||||||
|
public async Task<ActionResult<RestoreDrillResult>> RunRestoreDrill(
|
||||||
|
string backupId,
|
||||||
|
RestoreDrillRequest request,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
if (!request.Confirmation.Equals(
|
||||||
|
"RESTORE_DRILL",
|
||||||
|
StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
return ValidationProblem(
|
||||||
|
"恢复演练必须明确确认,且不会覆盖当前业务数据库。");
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
return Ok(await backupService.RunRestoreDrillAsync(
|
||||||
|
backupId,
|
||||||
|
cancellationToken));
|
||||||
|
}
|
||||||
|
catch (FileNotFoundException)
|
||||||
|
{
|
||||||
|
return NotFound(new ProblemDetails
|
||||||
|
{
|
||||||
|
Title = "备份不存在",
|
||||||
|
Detail = "指定备份不存在或其文件已被移除。",
|
||||||
|
Status = StatusCodes.Status404NotFound
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<IReadOnlyCollection<OperationalAlert>> BuildAlertsAsync(
|
||||||
|
OperationalHealthSnapshot health,
|
||||||
|
IReadOnlyCollection<BackupArtifact> backups,
|
||||||
|
int serverErrorCount,
|
||||||
|
int failedJobCount,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var alerts = health.Components
|
||||||
|
.Where(x => x.Status != "healthy")
|
||||||
|
.Select(x => new OperationalAlert(
|
||||||
|
$"health-{x.Key}",
|
||||||
|
x.Status == "unhealthy" ? "critical" : "warning",
|
||||||
|
"health",
|
||||||
|
$"{x.Label}状态异常",
|
||||||
|
x.Detail,
|
||||||
|
health.CheckedAt))
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
if (serverErrorCount > 0)
|
||||||
|
{
|
||||||
|
var latest = await db.AuditLogs.AsNoTracking()
|
||||||
|
.Where(x =>
|
||||||
|
x.CreatedAt >= DateTime.UtcNow.AddHours(-24) &&
|
||||||
|
x.StatusCode >= 500)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.Select(x => new { x.Path, x.StatusCode, x.CreatedAt })
|
||||||
|
.FirstOrDefaultAsync(cancellationToken);
|
||||||
|
alerts.Add(new OperationalAlert(
|
||||||
|
"http-5xx",
|
||||||
|
"critical",
|
||||||
|
"audit",
|
||||||
|
$"过去 24 小时发生 {serverErrorCount} 次服务端错误",
|
||||||
|
latest is null
|
||||||
|
? "请检查服务日志定位异常。"
|
||||||
|
: $"最近一次为 {latest.StatusCode} {latest.Path}。",
|
||||||
|
latest?.CreatedAt ?? DateTime.UtcNow));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (failedJobCount > 0)
|
||||||
|
{
|
||||||
|
alerts.Add(new OperationalAlert(
|
||||||
|
"failed-jobs",
|
||||||
|
"critical",
|
||||||
|
"jobs",
|
||||||
|
$"最近 7 天有 {failedJobCount} 个后台任务失败",
|
||||||
|
"任务已停止或达到重试上限,请在失败任务中查看错误详情。",
|
||||||
|
DateTime.UtcNow));
|
||||||
|
}
|
||||||
|
|
||||||
|
var retryingFailures = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||||
|
.CountAsync(
|
||||||
|
x => x.State != BackgroundJobOutboxState.Completed &&
|
||||||
|
x.LastError != null,
|
||||||
|
cancellationToken);
|
||||||
|
if (retryingFailures > 0)
|
||||||
|
{
|
||||||
|
alerts.Add(new OperationalAlert(
|
||||||
|
"retrying-jobs",
|
||||||
|
"warning",
|
||||||
|
"jobs",
|
||||||
|
$"{retryingFailures} 个后台任务正在错误重试",
|
||||||
|
"任务队列仍会自动重试;若持续出现,请检查依赖服务与任务参数。",
|
||||||
|
DateTime.UtcNow));
|
||||||
|
}
|
||||||
|
|
||||||
|
var latestBackup = backups.FirstOrDefault();
|
||||||
|
if (latestBackup is null)
|
||||||
|
{
|
||||||
|
alerts.Add(new OperationalAlert(
|
||||||
|
"backup-missing",
|
||||||
|
"critical",
|
||||||
|
"backup",
|
||||||
|
"尚无可验证的数据库备份",
|
||||||
|
"立即创建首个备份,并在创建后执行一次隔离恢复演练。",
|
||||||
|
DateTime.UtcNow));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
var ageHours = (DateTime.UtcNow - latestBackup.CreatedAt).TotalHours;
|
||||||
|
if (ageHours > options.BackupWarningHours)
|
||||||
|
{
|
||||||
|
alerts.Add(new OperationalAlert(
|
||||||
|
"backup-stale",
|
||||||
|
"warning",
|
||||||
|
"backup",
|
||||||
|
$"最近备份已超过 {options.BackupWarningHours} 小时",
|
||||||
|
$"最近备份创建于 {latestBackup.CreatedAt:u}。",
|
||||||
|
latestBackup.CreatedAt));
|
||||||
|
}
|
||||||
|
if (latestBackup.LastDrillSucceeded == false)
|
||||||
|
{
|
||||||
|
alerts.Add(new OperationalAlert(
|
||||||
|
"restore-drill-failed",
|
||||||
|
"critical",
|
||||||
|
"backup",
|
||||||
|
"最近一次恢复演练失败",
|
||||||
|
latestBackup.LastDrillDetail ?? "请重新运行演练并检查数据库工具日志。",
|
||||||
|
latestBackup.LastDrillAt ?? latestBackup.CreatedAt));
|
||||||
|
}
|
||||||
|
else if (!latestBackup.LastDrillAt.HasValue)
|
||||||
|
{
|
||||||
|
alerts.Add(new OperationalAlert(
|
||||||
|
"restore-drill-missing",
|
||||||
|
"warning",
|
||||||
|
"backup",
|
||||||
|
"最近备份尚未完成恢复演练",
|
||||||
|
"恢复演练只写入隔离数据库,不会覆盖当前业务数据。",
|
||||||
|
latestBackup.CreatedAt));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return alerts
|
||||||
|
.OrderBy(x => x.Severity == "critical" ? 0 : 1)
|
||||||
|
.ThenByDescending(x => x.OccurredAt)
|
||||||
|
.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<int> CountFailedJobsAsync(
|
||||||
|
DateTime from,
|
||||||
|
CancellationToken cancellationToken) =>
|
||||||
|
await db.AutomaticScheduleJobs.AsNoTracking()
|
||||||
|
.CountAsync(
|
||||||
|
x => x.Status == AutomaticScheduleJobStatus.Failed &&
|
||||||
|
x.CreatedAt >= from,
|
||||||
|
cancellationToken) +
|
||||||
|
await db.SchedulePublishJobs.AsNoTracking()
|
||||||
|
.CountAsync(
|
||||||
|
x => x.Status == SchedulePublishJobStatus.Failed &&
|
||||||
|
x.CreatedAt >= from,
|
||||||
|
cancellationToken) +
|
||||||
|
await db.MakeupExamAutoJobs.AsNoTracking()
|
||||||
|
.CountAsync(
|
||||||
|
x => x.Status == MakeupExamAutoJobStatus.Failed &&
|
||||||
|
x.CreatedAt >= from,
|
||||||
|
cancellationToken);
|
||||||
|
|
||||||
|
private ActionResult? ValidatePaging(int page, int pageSize)
|
||||||
|
{
|
||||||
|
if (page is < 1 or > 100000 || pageSize is < 1 or > 100)
|
||||||
|
{
|
||||||
|
return ValidationProblem(
|
||||||
|
"页码必须在 1 到 100000 之间,每页数量必须在 1 到 100 之间。");
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private ActionResult? ValidateRange(DateTime? from, DateTime? to)
|
||||||
|
{
|
||||||
|
if (from.HasValue && to.HasValue && from.Value > to.Value)
|
||||||
|
return ValidationProblem("开始时间不能晚于结束时间。");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string? NormalizeJobKind(string? kind)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(kind)) return null;
|
||||||
|
return kind.Trim() switch
|
||||||
|
{
|
||||||
|
"AutomaticSchedule" => "AutomaticSchedule",
|
||||||
|
"SchedulePublish" => "SchedulePublish",
|
||||||
|
"MakeupExamAuto" => "MakeupExamAuto",
|
||||||
|
_ => null
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string SafeMessage(Exception exception)
|
||||||
|
{
|
||||||
|
var message = exception.GetBaseException().Message;
|
||||||
|
return message.Length <= 500 ? message : message[..500];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed record AuditLogItem(
|
||||||
|
Guid Id,
|
||||||
|
string? UserName,
|
||||||
|
string Method,
|
||||||
|
string Path,
|
||||||
|
int StatusCode,
|
||||||
|
string? IpAddress,
|
||||||
|
DateTime CreatedAt);
|
||||||
|
|
||||||
|
public sealed record FailedBackgroundJobItem(
|
||||||
|
Guid Id,
|
||||||
|
string Kind,
|
||||||
|
string KindLabel,
|
||||||
|
string Context,
|
||||||
|
string ErrorMessage,
|
||||||
|
DateTime CreatedAt,
|
||||||
|
DateTime? StartedAt,
|
||||||
|
DateTime? CompletedAt,
|
||||||
|
int ProcessingAttempts);
|
||||||
|
|
||||||
|
public sealed record OperationalAlert(
|
||||||
|
string Id,
|
||||||
|
string Severity,
|
||||||
|
string Source,
|
||||||
|
string Title,
|
||||||
|
string Detail,
|
||||||
|
DateTime OccurredAt);
|
||||||
|
|
||||||
|
public sealed record OperationsCounters(
|
||||||
|
int AuditEvents24Hours,
|
||||||
|
int ServerErrors24Hours,
|
||||||
|
int FailedJobs7Days,
|
||||||
|
int CriticalAlerts);
|
||||||
|
|
||||||
|
public sealed record OperationsSummary(
|
||||||
|
DateTime GeneratedAt,
|
||||||
|
OperationalHealthSnapshot Health,
|
||||||
|
OperationsCounters Counters,
|
||||||
|
IReadOnlyCollection<OperationalAlert> Alerts,
|
||||||
|
BackupArtifact? LatestBackup);
|
||||||
|
|
||||||
|
public sealed record CreateBackupRequest([MaxLength(200)] string? Note);
|
||||||
|
|
||||||
|
public sealed record RestoreDrillRequest([Required] string Confirmation);
|
||||||
@@ -0,0 +1,584 @@
|
|||||||
|
using System.Diagnostics;
|
||||||
|
using System.Security.Cryptography;
|
||||||
|
using System.Text.Json;
|
||||||
|
using Jiaowu.Api.Infrastructure.Persistence;
|
||||||
|
using Microsoft.Data.Sqlite;
|
||||||
|
using MySql.Data.MySqlClient;
|
||||||
|
|
||||||
|
namespace Jiaowu.Api.Infrastructure.Operations;
|
||||||
|
|
||||||
|
public sealed record BackupArtifact(
|
||||||
|
string Id,
|
||||||
|
string FileName,
|
||||||
|
string Provider,
|
||||||
|
DateTime CreatedAt,
|
||||||
|
long SizeBytes,
|
||||||
|
string Sha256,
|
||||||
|
string? Note,
|
||||||
|
DateTime? LastDrillAt,
|
||||||
|
bool? LastDrillSucceeded,
|
||||||
|
string? LastDrillDetail,
|
||||||
|
long? LastDrillDurationMilliseconds);
|
||||||
|
|
||||||
|
public sealed record RestoreDrillResult(
|
||||||
|
string BackupId,
|
||||||
|
bool Succeeded,
|
||||||
|
DateTime CompletedAt,
|
||||||
|
string Detail,
|
||||||
|
long DurationMilliseconds,
|
||||||
|
int? TableCount);
|
||||||
|
|
||||||
|
public sealed class DatabaseBackupService(
|
||||||
|
DatabaseOptions databaseOptions,
|
||||||
|
OperationsOptions options,
|
||||||
|
IConfiguration configuration,
|
||||||
|
IHostEnvironment environment,
|
||||||
|
ILogger<DatabaseBackupService> logger)
|
||||||
|
{
|
||||||
|
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||||
|
{
|
||||||
|
WriteIndented = true
|
||||||
|
};
|
||||||
|
|
||||||
|
private readonly SemaphoreSlim operationLock = new(1, 1);
|
||||||
|
private readonly string backupDirectory = ResolveBackupDirectory(
|
||||||
|
options.BackupDirectory,
|
||||||
|
environment.ContentRootPath);
|
||||||
|
|
||||||
|
public async Task<IReadOnlyCollection<BackupArtifact>> ListAsync(
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(backupDirectory);
|
||||||
|
var items = new List<BackupArtifact>();
|
||||||
|
foreach (var metadataPath in Directory.EnumerateFiles(
|
||||||
|
backupDirectory,
|
||||||
|
"*.metadata.json",
|
||||||
|
SearchOption.TopDirectoryOnly))
|
||||||
|
{
|
||||||
|
cancellationToken.ThrowIfCancellationRequested();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await using var stream = File.OpenRead(metadataPath);
|
||||||
|
var artifact = await JsonSerializer.DeserializeAsync<BackupArtifact>(
|
||||||
|
stream,
|
||||||
|
JsonOptions,
|
||||||
|
cancellationToken);
|
||||||
|
if (artifact is not null &&
|
||||||
|
File.Exists(Path.Combine(backupDirectory, artifact.FileName)))
|
||||||
|
{
|
||||||
|
items.Add(artifact);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception exception) when (
|
||||||
|
exception is IOException or UnauthorizedAccessException or JsonException)
|
||||||
|
{
|
||||||
|
logger.LogWarning(
|
||||||
|
exception,
|
||||||
|
"Unable to read backup metadata {MetadataFile}.",
|
||||||
|
Path.GetFileName(metadataPath));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return items
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<BackupArtifact> CreateAsync(
|
||||||
|
string? note,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await operationLock.WaitAsync(cancellationToken);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(backupDirectory);
|
||||||
|
var createdAt = DateTime.UtcNow;
|
||||||
|
var id = $"{createdAt:yyyyMMddHHmmss}-{Guid.NewGuid():N}"[..29];
|
||||||
|
var provider = NormalizeProvider(databaseOptions.Provider);
|
||||||
|
var extension = provider == "SQLite" ? ".sqlite" : ".sql";
|
||||||
|
var fileName = $"jiaowu-{id}{extension}";
|
||||||
|
var backupPath = Path.Combine(backupDirectory, fileName);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (provider == "SQLite")
|
||||||
|
await CreateSqliteBackupAsync(backupPath, cancellationToken);
|
||||||
|
else
|
||||||
|
await CreateMySqlBackupAsync(backupPath, cancellationToken);
|
||||||
|
|
||||||
|
var fileInfo = new FileInfo(backupPath);
|
||||||
|
var artifact = new BackupArtifact(
|
||||||
|
id,
|
||||||
|
fileName,
|
||||||
|
provider,
|
||||||
|
createdAt,
|
||||||
|
fileInfo.Length,
|
||||||
|
await ComputeHashAsync(backupPath, cancellationToken),
|
||||||
|
NormalizeNote(note),
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null);
|
||||||
|
await WriteMetadataAsync(artifact, cancellationToken);
|
||||||
|
return artifact;
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
if (File.Exists(backupPath))
|
||||||
|
File.Delete(backupPath);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
operationLock.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<RestoreDrillResult> RunRestoreDrillAsync(
|
||||||
|
string backupId,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await operationLock.WaitAsync(cancellationToken);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var artifact = (await ListAsync(cancellationToken))
|
||||||
|
.SingleOrDefault(x => x.Id.Equals(backupId, StringComparison.Ordinal));
|
||||||
|
if (artifact is null)
|
||||||
|
throw new FileNotFoundException("未找到指定备份。");
|
||||||
|
|
||||||
|
var backupPath = Path.Combine(backupDirectory, artifact.FileName);
|
||||||
|
var actualHash = await ComputeHashAsync(backupPath, cancellationToken);
|
||||||
|
if (!CryptographicOperations.FixedTimeEquals(
|
||||||
|
Convert.FromHexString(artifact.Sha256),
|
||||||
|
Convert.FromHexString(actualHash)))
|
||||||
|
{
|
||||||
|
var damaged = await CompleteDrillAsync(
|
||||||
|
artifact,
|
||||||
|
false,
|
||||||
|
"备份文件校验和不一致,恢复演练已中止。",
|
||||||
|
0,
|
||||||
|
null,
|
||||||
|
cancellationToken);
|
||||||
|
return damaged;
|
||||||
|
}
|
||||||
|
|
||||||
|
var stopwatch = Stopwatch.StartNew();
|
||||||
|
RestoreDrillResult result;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var tableCount = artifact.Provider == "SQLite"
|
||||||
|
? await DrillSqliteAsync(backupPath, cancellationToken)
|
||||||
|
: await DrillMySqlAsync(backupPath, cancellationToken);
|
||||||
|
stopwatch.Stop();
|
||||||
|
result = await CompleteDrillAsync(
|
||||||
|
artifact,
|
||||||
|
true,
|
||||||
|
$"已在隔离数据库完成恢复并通过完整性检查,共发现 {tableCount} 张业务表。",
|
||||||
|
stopwatch.ElapsedMilliseconds,
|
||||||
|
tableCount,
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||||
|
{
|
||||||
|
stopwatch.Stop();
|
||||||
|
logger.LogError(
|
||||||
|
exception,
|
||||||
|
"Restore drill failed for backup {BackupId}.",
|
||||||
|
artifact.Id);
|
||||||
|
result = await CompleteDrillAsync(
|
||||||
|
artifact,
|
||||||
|
false,
|
||||||
|
$"恢复演练失败:{SafeMessage(exception)}",
|
||||||
|
stopwatch.ElapsedMilliseconds,
|
||||||
|
null,
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
operationLock.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task CreateSqliteBackupAsync(
|
||||||
|
string backupPath,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var sourceBuilder = new SqliteConnectionStringBuilder(
|
||||||
|
configuration.GetConnectionString("SQLite")
|
||||||
|
?? throw new InvalidOperationException("缺少 ConnectionStrings:SQLite。"));
|
||||||
|
if (!Path.IsPathRooted(sourceBuilder.DataSource))
|
||||||
|
{
|
||||||
|
sourceBuilder.DataSource = Path.GetFullPath(
|
||||||
|
sourceBuilder.DataSource,
|
||||||
|
environment.ContentRootPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
var destinationBuilder = new SqliteConnectionStringBuilder
|
||||||
|
{
|
||||||
|
DataSource = backupPath,
|
||||||
|
Mode = SqliteOpenMode.ReadWriteCreate,
|
||||||
|
Pooling = false
|
||||||
|
};
|
||||||
|
await using var source = new SqliteConnection(sourceBuilder.ConnectionString);
|
||||||
|
await using var destination = new SqliteConnection(destinationBuilder.ConnectionString);
|
||||||
|
await source.OpenAsync(cancellationToken);
|
||||||
|
await destination.OpenAsync(cancellationToken);
|
||||||
|
source.BackupDatabase(destination);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task CreateMySqlBackupAsync(
|
||||||
|
string backupPath,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var connection = GetMySqlConnectionBuilder();
|
||||||
|
var arguments = new List<string>
|
||||||
|
{
|
||||||
|
"--protocol=tcp",
|
||||||
|
$"--host={connection.Server}",
|
||||||
|
$"--port={connection.Port}",
|
||||||
|
$"--user={connection.UserID}",
|
||||||
|
"--single-transaction",
|
||||||
|
"--quick",
|
||||||
|
"--routines",
|
||||||
|
"--triggers",
|
||||||
|
"--events",
|
||||||
|
"--hex-blob",
|
||||||
|
"--default-character-set=utf8mb4"
|
||||||
|
};
|
||||||
|
arguments.AddRange(options.MySqlAdditionalArguments);
|
||||||
|
arguments.Add(connection.Database);
|
||||||
|
|
||||||
|
await using var output = new FileStream(
|
||||||
|
backupPath,
|
||||||
|
FileMode.CreateNew,
|
||||||
|
FileAccess.Write,
|
||||||
|
FileShare.None,
|
||||||
|
81920,
|
||||||
|
FileOptions.Asynchronous);
|
||||||
|
await RunToolAsync(
|
||||||
|
options.MySqlDumpPath,
|
||||||
|
arguments,
|
||||||
|
connection.Password,
|
||||||
|
standardInput: null,
|
||||||
|
standardOutput: output,
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<int> DrillSqliteAsync(
|
||||||
|
string backupPath,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var drillPath = Path.Combine(
|
||||||
|
Path.GetDirectoryName(backupPath)!,
|
||||||
|
$".restore-drill-{Guid.NewGuid():N}.sqlite");
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var sourceBuilder = new SqliteConnectionStringBuilder
|
||||||
|
{
|
||||||
|
DataSource = backupPath,
|
||||||
|
Mode = SqliteOpenMode.ReadOnly,
|
||||||
|
Pooling = false
|
||||||
|
};
|
||||||
|
var drillBuilder = new SqliteConnectionStringBuilder
|
||||||
|
{
|
||||||
|
DataSource = drillPath,
|
||||||
|
Mode = SqliteOpenMode.ReadWriteCreate,
|
||||||
|
Pooling = false
|
||||||
|
};
|
||||||
|
await using var source = new SqliteConnection(sourceBuilder.ConnectionString);
|
||||||
|
await using var drill = new SqliteConnection(drillBuilder.ConnectionString);
|
||||||
|
await source.OpenAsync(cancellationToken);
|
||||||
|
await drill.OpenAsync(cancellationToken);
|
||||||
|
source.BackupDatabase(drill);
|
||||||
|
|
||||||
|
await using var integrity = drill.CreateCommand();
|
||||||
|
integrity.CommandText = "PRAGMA integrity_check;";
|
||||||
|
var integrityResult = Convert.ToString(
|
||||||
|
await integrity.ExecuteScalarAsync(cancellationToken));
|
||||||
|
if (!string.Equals(integrityResult, "ok", StringComparison.OrdinalIgnoreCase))
|
||||||
|
throw new InvalidDataException($"SQLite 完整性检查返回 {integrityResult ?? "空结果"}。");
|
||||||
|
|
||||||
|
await using var tables = drill.CreateCommand();
|
||||||
|
tables.CommandText =
|
||||||
|
"SELECT COUNT(*) FROM sqlite_master " +
|
||||||
|
"WHERE type = 'table' AND name NOT LIKE 'sqlite_%';";
|
||||||
|
return Convert.ToInt32(await tables.ExecuteScalarAsync(cancellationToken));
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (File.Exists(drillPath))
|
||||||
|
File.Delete(drillPath);
|
||||||
|
if (File.Exists($"{drillPath}-shm"))
|
||||||
|
File.Delete($"{drillPath}-shm");
|
||||||
|
if (File.Exists($"{drillPath}-wal"))
|
||||||
|
File.Delete($"{drillPath}-wal");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<int> DrillMySqlAsync(
|
||||||
|
string backupPath,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var connection = GetMySqlConnectionBuilder();
|
||||||
|
var drillDatabase = $"jiaowu_restore_drill_{DateTime.UtcNow:yyyyMMddHHmmss}_" +
|
||||||
|
Guid.NewGuid().ToString("N")[..8];
|
||||||
|
var adminBuilder = new MySqlConnectionStringBuilder(connection.ConnectionString)
|
||||||
|
{
|
||||||
|
Database = ""
|
||||||
|
};
|
||||||
|
|
||||||
|
await using var admin = new MySqlConnection(adminBuilder.ConnectionString);
|
||||||
|
await admin.OpenAsync(cancellationToken);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await using (var create = admin.CreateCommand())
|
||||||
|
{
|
||||||
|
create.CommandText =
|
||||||
|
$"CREATE DATABASE `{drillDatabase}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;";
|
||||||
|
await create.ExecuteNonQueryAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
var arguments = new List<string>
|
||||||
|
{
|
||||||
|
"--protocol=tcp",
|
||||||
|
$"--host={connection.Server}",
|
||||||
|
$"--port={connection.Port}",
|
||||||
|
$"--user={connection.UserID}",
|
||||||
|
"--default-character-set=utf8mb4"
|
||||||
|
};
|
||||||
|
arguments.AddRange(options.MySqlAdditionalArguments);
|
||||||
|
arguments.Add($"--database={drillDatabase}");
|
||||||
|
await using (var input = new FileStream(
|
||||||
|
backupPath,
|
||||||
|
FileMode.Open,
|
||||||
|
FileAccess.Read,
|
||||||
|
FileShare.Read,
|
||||||
|
81920,
|
||||||
|
FileOptions.Asynchronous))
|
||||||
|
{
|
||||||
|
await RunToolAsync(
|
||||||
|
options.MySqlClientPath,
|
||||||
|
arguments,
|
||||||
|
connection.Password,
|
||||||
|
input,
|
||||||
|
standardOutput: null,
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
await using var count = admin.CreateCommand();
|
||||||
|
count.CommandText =
|
||||||
|
"SELECT COUNT(*) FROM information_schema.tables " +
|
||||||
|
"WHERE table_schema = @schema AND table_type = 'BASE TABLE';";
|
||||||
|
count.Parameters.AddWithValue("@schema", drillDatabase);
|
||||||
|
return Convert.ToInt32(await count.ExecuteScalarAsync(cancellationToken));
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await using var drop = admin.CreateCommand();
|
||||||
|
drop.CommandText = $"DROP DATABASE IF EXISTS `{drillDatabase}`;";
|
||||||
|
await drop.ExecuteNonQueryAsync(CancellationToken.None);
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
logger.LogCritical(
|
||||||
|
exception,
|
||||||
|
"Unable to remove isolated restore drill database {DatabaseName}.",
|
||||||
|
drillDatabase);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task RunToolAsync(
|
||||||
|
string executable,
|
||||||
|
IReadOnlyCollection<string> arguments,
|
||||||
|
string password,
|
||||||
|
Stream? standardInput,
|
||||||
|
Stream? standardOutput,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var startInfo = new ProcessStartInfo
|
||||||
|
{
|
||||||
|
FileName = executable,
|
||||||
|
UseShellExecute = false,
|
||||||
|
RedirectStandardError = true,
|
||||||
|
RedirectStandardInput = standardInput is not null,
|
||||||
|
RedirectStandardOutput = standardOutput is not null,
|
||||||
|
CreateNoWindow = true
|
||||||
|
};
|
||||||
|
foreach (var argument in arguments)
|
||||||
|
startInfo.ArgumentList.Add(argument);
|
||||||
|
if (!string.IsNullOrEmpty(password))
|
||||||
|
startInfo.Environment["MYSQL_PWD"] = password;
|
||||||
|
|
||||||
|
using var process = Process.Start(startInfo)
|
||||||
|
?? throw new InvalidOperationException($"无法启动数据库工具 {executable}。");
|
||||||
|
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||||
|
timeout.CancelAfter(TimeSpan.FromMinutes(options.ToolTimeoutMinutes));
|
||||||
|
var errorTask = process.StandardError.ReadToEndAsync(timeout.Token);
|
||||||
|
var inputTask = standardInput is null
|
||||||
|
? Task.CompletedTask
|
||||||
|
: CopyInputAsync(standardInput, process.StandardInput.BaseStream, timeout.Token);
|
||||||
|
var outputTask = standardOutput is null
|
||||||
|
? Task.CompletedTask
|
||||||
|
: process.StandardOutput.BaseStream.CopyToAsync(
|
||||||
|
standardOutput,
|
||||||
|
timeout.Token);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await Task.WhenAll(
|
||||||
|
process.WaitForExitAsync(timeout.Token),
|
||||||
|
inputTask,
|
||||||
|
outputTask);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException)
|
||||||
|
{
|
||||||
|
if (!process.HasExited)
|
||||||
|
process.Kill(entireProcessTree: true);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
|
||||||
|
var error = await errorTask;
|
||||||
|
if (process.ExitCode != 0)
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"数据库工具执行失败(退出码 {process.ExitCode}):{TrimToolError(error)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task CopyInputAsync(
|
||||||
|
Stream input,
|
||||||
|
Stream processInput,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await input.CopyToAsync(processInput, cancellationToken);
|
||||||
|
await processInput.FlushAsync(cancellationToken);
|
||||||
|
processInput.Close();
|
||||||
|
}
|
||||||
|
|
||||||
|
private MySqlConnectionStringBuilder GetMySqlConnectionBuilder()
|
||||||
|
{
|
||||||
|
var value = configuration.GetConnectionString("OperationsMySql");
|
||||||
|
if (string.IsNullOrWhiteSpace(value))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
"MySQL 备份与恢复演练必须配置独立的 " +
|
||||||
|
"ConnectionStrings:OperationsMySql 运维账号,不能复用日常业务账号。");
|
||||||
|
}
|
||||||
|
var builder = new MySqlConnectionStringBuilder(value);
|
||||||
|
if (string.IsNullOrWhiteSpace(builder.Database))
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
"OperationsMySql 连接字符串未指定业务数据库名称。");
|
||||||
|
return builder;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<RestoreDrillResult> CompleteDrillAsync(
|
||||||
|
BackupArtifact artifact,
|
||||||
|
bool succeeded,
|
||||||
|
string detail,
|
||||||
|
long durationMilliseconds,
|
||||||
|
int? tableCount,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var completedAt = DateTime.UtcNow;
|
||||||
|
var updated = artifact with
|
||||||
|
{
|
||||||
|
LastDrillAt = completedAt,
|
||||||
|
LastDrillSucceeded = succeeded,
|
||||||
|
LastDrillDetail = detail,
|
||||||
|
LastDrillDurationMilliseconds = durationMilliseconds
|
||||||
|
};
|
||||||
|
await WriteMetadataAsync(updated, cancellationToken);
|
||||||
|
return new RestoreDrillResult(
|
||||||
|
artifact.Id,
|
||||||
|
succeeded,
|
||||||
|
completedAt,
|
||||||
|
detail,
|
||||||
|
durationMilliseconds,
|
||||||
|
tableCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task WriteMetadataAsync(
|
||||||
|
BackupArtifact artifact,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var metadataPath = Path.Combine(
|
||||||
|
backupDirectory,
|
||||||
|
$"{artifact.Id}.metadata.json");
|
||||||
|
var temporaryPath = $"{metadataPath}.{Guid.NewGuid():N}.tmp";
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await using (var stream = new FileStream(
|
||||||
|
temporaryPath,
|
||||||
|
FileMode.CreateNew,
|
||||||
|
FileAccess.Write,
|
||||||
|
FileShare.None,
|
||||||
|
16384,
|
||||||
|
FileOptions.Asynchronous))
|
||||||
|
{
|
||||||
|
await JsonSerializer.SerializeAsync(
|
||||||
|
stream,
|
||||||
|
artifact,
|
||||||
|
JsonOptions,
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
File.Move(temporaryPath, metadataPath, overwrite: true);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (File.Exists(temporaryPath))
|
||||||
|
File.Delete(temporaryPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<string> ComputeHashAsync(
|
||||||
|
string path,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await using var stream = File.OpenRead(path);
|
||||||
|
return Convert.ToHexString(
|
||||||
|
await SHA256.HashDataAsync(stream, cancellationToken));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string ResolveBackupDirectory(
|
||||||
|
string configuredPath,
|
||||||
|
string contentRoot)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(configuredPath))
|
||||||
|
throw new InvalidOperationException("Operations:BackupDirectory 不能为空。");
|
||||||
|
return Path.IsPathRooted(configuredPath)
|
||||||
|
? Path.GetFullPath(configuredPath)
|
||||||
|
: Path.GetFullPath(configuredPath, contentRoot);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string NormalizeProvider(string provider) =>
|
||||||
|
provider.Equals("SQLite", StringComparison.OrdinalIgnoreCase)
|
||||||
|
? "SQLite"
|
||||||
|
: provider.Equals("MySql", StringComparison.OrdinalIgnoreCase)
|
||||||
|
? "MySql"
|
||||||
|
: throw new InvalidOperationException($"不支持数据库 Provider '{provider}'。");
|
||||||
|
|
||||||
|
private static string? NormalizeNote(string? note)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(note)) return null;
|
||||||
|
var trimmed = note.Trim();
|
||||||
|
return trimmed.Length <= 200 ? trimmed : trimmed[..200];
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string SafeMessage(Exception exception)
|
||||||
|
{
|
||||||
|
var message = exception.GetBaseException().Message;
|
||||||
|
return message.Length <= 500 ? message : message[..500];
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string TrimToolError(string error)
|
||||||
|
{
|
||||||
|
var trimmed = error.Trim();
|
||||||
|
if (trimmed.Length == 0) return "未返回错误详情。";
|
||||||
|
return trimmed.Length <= 500 ? trimmed : trimmed[..500];
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,200 @@
|
|||||||
|
using System.Diagnostics;
|
||||||
|
using Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||||
|
using Jiaowu.Api.Infrastructure.Caching;
|
||||||
|
using Jiaowu.Api.Infrastructure.Persistence;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.Extensions.Caching.Distributed;
|
||||||
|
|
||||||
|
namespace Jiaowu.Api.Infrastructure.Operations;
|
||||||
|
|
||||||
|
public sealed record OperationalComponentHealth(
|
||||||
|
string Key,
|
||||||
|
string Label,
|
||||||
|
string Status,
|
||||||
|
string Backend,
|
||||||
|
long? LatencyMilliseconds,
|
||||||
|
string Detail);
|
||||||
|
|
||||||
|
public sealed record OperationalHealthSnapshot(
|
||||||
|
DateTime CheckedAt,
|
||||||
|
string OverallStatus,
|
||||||
|
IReadOnlyCollection<OperationalComponentHealth> Components,
|
||||||
|
BackgroundJobBacklogSnapshot? Backlog);
|
||||||
|
|
||||||
|
public sealed class OperationalHealthService(
|
||||||
|
AppDbContext db,
|
||||||
|
IServiceProvider services,
|
||||||
|
IBackgroundJobTransport transport,
|
||||||
|
BackgroundJobMonitoringService monitoring,
|
||||||
|
DatabaseOptions databaseOptions,
|
||||||
|
AppCacheOptions cacheOptions,
|
||||||
|
IConfiguration configuration)
|
||||||
|
{
|
||||||
|
public async Task<OperationalHealthSnapshot> CheckAsync(
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var database = await CheckDatabaseAsync(cancellationToken);
|
||||||
|
var cache = await CheckCacheAsync(cancellationToken);
|
||||||
|
var (messaging, backlog) = await CheckMessagingAsync(cancellationToken);
|
||||||
|
var components = new[] { database, cache, messaging };
|
||||||
|
var overall = components.Any(x => x.Status == "unhealthy")
|
||||||
|
? "unhealthy"
|
||||||
|
: components.Any(x => x.Status == "warning")
|
||||||
|
? "warning"
|
||||||
|
: "healthy";
|
||||||
|
return new OperationalHealthSnapshot(
|
||||||
|
DateTime.UtcNow,
|
||||||
|
overall,
|
||||||
|
components,
|
||||||
|
backlog);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<OperationalComponentHealth> CheckDatabaseAsync(
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var stopwatch = Stopwatch.StartNew();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var canConnect = await db.Database.CanConnectAsync(cancellationToken);
|
||||||
|
stopwatch.Stop();
|
||||||
|
return canConnect
|
||||||
|
? new OperationalComponentHealth(
|
||||||
|
"database",
|
||||||
|
"数据库",
|
||||||
|
"healthy",
|
||||||
|
databaseOptions.Provider,
|
||||||
|
stopwatch.ElapsedMilliseconds,
|
||||||
|
"连接与基础查询正常。")
|
||||||
|
: new OperationalComponentHealth(
|
||||||
|
"database",
|
||||||
|
"数据库",
|
||||||
|
"unhealthy",
|
||||||
|
databaseOptions.Provider,
|
||||||
|
stopwatch.ElapsedMilliseconds,
|
||||||
|
"无法建立数据库连接。");
|
||||||
|
}
|
||||||
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||||
|
{
|
||||||
|
stopwatch.Stop();
|
||||||
|
return new OperationalComponentHealth(
|
||||||
|
"database",
|
||||||
|
"数据库",
|
||||||
|
"unhealthy",
|
||||||
|
databaseOptions.Provider,
|
||||||
|
stopwatch.ElapsedMilliseconds,
|
||||||
|
SafeMessage(exception));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<OperationalComponentHealth> CheckCacheAsync(
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
if (!cacheOptions.Enabled)
|
||||||
|
{
|
||||||
|
return new OperationalComponentHealth(
|
||||||
|
"cache",
|
||||||
|
"缓存",
|
||||||
|
"warning",
|
||||||
|
"disabled",
|
||||||
|
null,
|
||||||
|
"缓存已通过配置关闭,所有查询将直接访问数据源。");
|
||||||
|
}
|
||||||
|
|
||||||
|
var hasRedisConfiguration = !string.IsNullOrWhiteSpace(
|
||||||
|
configuration.GetConnectionString("Redis"));
|
||||||
|
var distributedCache = services.GetService<IDistributedCache>();
|
||||||
|
if (!hasRedisConfiguration || distributedCache is null)
|
||||||
|
{
|
||||||
|
return new OperationalComponentHealth(
|
||||||
|
"cache",
|
||||||
|
"缓存",
|
||||||
|
"healthy",
|
||||||
|
"memory",
|
||||||
|
null,
|
||||||
|
"使用进程内混合缓存;服务重启后缓存会自然重建。");
|
||||||
|
}
|
||||||
|
|
||||||
|
var stopwatch = Stopwatch.StartNew();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await distributedCache.GetAsync(
|
||||||
|
"jiaowu:operations:health-probe",
|
||||||
|
cancellationToken);
|
||||||
|
stopwatch.Stop();
|
||||||
|
return new OperationalComponentHealth(
|
||||||
|
"cache",
|
||||||
|
"缓存",
|
||||||
|
"healthy",
|
||||||
|
"redis",
|
||||||
|
stopwatch.ElapsedMilliseconds,
|
||||||
|
"Redis 连接与读取探针正常。");
|
||||||
|
}
|
||||||
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||||
|
{
|
||||||
|
stopwatch.Stop();
|
||||||
|
return new OperationalComponentHealth(
|
||||||
|
"cache",
|
||||||
|
"缓存",
|
||||||
|
"unhealthy",
|
||||||
|
"redis",
|
||||||
|
stopwatch.ElapsedMilliseconds,
|
||||||
|
SafeMessage(exception));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<(OperationalComponentHealth Component,
|
||||||
|
BackgroundJobBacklogSnapshot? Backlog)> CheckMessagingAsync(
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var stopwatch = Stopwatch.StartNew();
|
||||||
|
BackgroundJobBacklogSnapshot? backlog = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var healthy = await transport.CheckHealthAsync(cancellationToken);
|
||||||
|
backlog = await monitoring.GetSnapshotAsync(cancellationToken);
|
||||||
|
stopwatch.Stop();
|
||||||
|
var stuck = backlog.ExpiredLeases > 0 ||
|
||||||
|
backlog.OldestUnfinishedAgeSeconds > 1800;
|
||||||
|
var status = !healthy
|
||||||
|
? "unhealthy"
|
||||||
|
: stuck
|
||||||
|
? "warning"
|
||||||
|
: "healthy";
|
||||||
|
var detail = !healthy
|
||||||
|
? "后台任务传输不可用。"
|
||||||
|
: stuck
|
||||||
|
? $"发现 {backlog.ExpiredLeases} 个过期租约,最早未完成任务已等待 " +
|
||||||
|
$"{Math.Round(backlog.OldestUnfinishedAgeSeconds ?? 0)} 秒。"
|
||||||
|
: $"待发布 {backlog.Pending + backlog.Publishing}," +
|
||||||
|
$"待处理 {backlog.Published + backlog.Processing}。";
|
||||||
|
return (
|
||||||
|
new OperationalComponentHealth(
|
||||||
|
"messaging",
|
||||||
|
"后台任务通道",
|
||||||
|
status,
|
||||||
|
transport.IsDurable ? "rabbitmq" : "memory",
|
||||||
|
stopwatch.ElapsedMilliseconds,
|
||||||
|
detail),
|
||||||
|
backlog);
|
||||||
|
}
|
||||||
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||||
|
{
|
||||||
|
stopwatch.Stop();
|
||||||
|
return (
|
||||||
|
new OperationalComponentHealth(
|
||||||
|
"messaging",
|
||||||
|
"后台任务通道",
|
||||||
|
"unhealthy",
|
||||||
|
transport.IsDurable ? "rabbitmq" : "memory",
|
||||||
|
stopwatch.ElapsedMilliseconds,
|
||||||
|
SafeMessage(exception)),
|
||||||
|
backlog);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string SafeMessage(Exception exception)
|
||||||
|
{
|
||||||
|
var message = exception.GetBaseException().Message;
|
||||||
|
return message.Length <= 300 ? message : message[..300];
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
namespace Jiaowu.Api.Infrastructure.Operations;
|
||||||
|
|
||||||
|
public sealed class OperationsOptions
|
||||||
|
{
|
||||||
|
public const string SectionName = "Operations";
|
||||||
|
|
||||||
|
public string BackupDirectory { get; set; } = "data/backups";
|
||||||
|
public int BackupWarningHours { get; set; } = 24;
|
||||||
|
public int ToolTimeoutMinutes { get; set; } = 30;
|
||||||
|
public string MySqlDumpPath { get; set; } = "mysqldump";
|
||||||
|
public string MySqlClientPath { get; set; } = "mysql";
|
||||||
|
public string[] MySqlAdditionalArguments { get; set; } = [];
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ using Jiaowu.Api.Infrastructure.Caching;
|
|||||||
using Jiaowu.Api.Infrastructure.Exams;
|
using Jiaowu.Api.Infrastructure.Exams;
|
||||||
using Jiaowu.Api.Infrastructure.Middleware;
|
using Jiaowu.Api.Infrastructure.Middleware;
|
||||||
using Jiaowu.Api.Infrastructure.OfficialDocuments;
|
using Jiaowu.Api.Infrastructure.OfficialDocuments;
|
||||||
|
using Jiaowu.Api.Infrastructure.Operations;
|
||||||
using Jiaowu.Api.Infrastructure.Persistence;
|
using Jiaowu.Api.Infrastructure.Persistence;
|
||||||
using Jiaowu.Api.Infrastructure.Scheduling;
|
using Jiaowu.Api.Infrastructure.Scheduling;
|
||||||
using Jiaowu.Api.Infrastructure.Timetables;
|
using Jiaowu.Api.Infrastructure.Timetables;
|
||||||
@@ -74,6 +75,9 @@ var officialDocumentOptions = builder.Configuration
|
|||||||
var backgroundJobOptions = builder.Configuration
|
var backgroundJobOptions = builder.Configuration
|
||||||
.GetSection(BackgroundJobOptions.SectionName)
|
.GetSection(BackgroundJobOptions.SectionName)
|
||||||
.Get<BackgroundJobOptions>() ?? new BackgroundJobOptions();
|
.Get<BackgroundJobOptions>() ?? new BackgroundJobOptions();
|
||||||
|
var operationsOptions = builder.Configuration
|
||||||
|
.GetSection(OperationsOptions.SectionName)
|
||||||
|
.Get<OperationsOptions>() ?? new OperationsOptions();
|
||||||
var rabbitMqOptions = builder.Configuration
|
var rabbitMqOptions = builder.Configuration
|
||||||
.GetSection(RabbitMqOptions.SectionName)
|
.GetSection(RabbitMqOptions.SectionName)
|
||||||
.Get<RabbitMqOptions>() ?? new RabbitMqOptions();
|
.Get<RabbitMqOptions>() ?? new RabbitMqOptions();
|
||||||
@@ -167,10 +171,25 @@ if (backgroundJobOptions.UsesRabbitMq &&
|
|||||||
"生产环境启用 RabbitMQ 时不能使用默认 guest 凭据。");
|
"生产环境启用 RabbitMQ 时不能使用默认 guest 凭据。");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (string.IsNullOrWhiteSpace(operationsOptions.BackupDirectory) ||
|
||||||
|
operationsOptions.BackupWarningHours is < 1 or > 8760 ||
|
||||||
|
operationsOptions.ToolTimeoutMinutes is < 1 or > 240 ||
|
||||||
|
string.IsNullOrWhiteSpace(operationsOptions.MySqlDumpPath) ||
|
||||||
|
string.IsNullOrWhiteSpace(operationsOptions.MySqlClientPath) ||
|
||||||
|
operationsOptions.MySqlAdditionalArguments.Length > 20 ||
|
||||||
|
operationsOptions.MySqlAdditionalArguments.Any(argument =>
|
||||||
|
string.IsNullOrWhiteSpace(argument) ||
|
||||||
|
argument.Length > 300 ||
|
||||||
|
!argument.StartsWith("--", StringComparison.Ordinal)))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Operations 运维与备份配置超出允许范围。");
|
||||||
|
}
|
||||||
|
|
||||||
builder.Services.AddSingleton(databaseOptions);
|
builder.Services.AddSingleton(databaseOptions);
|
||||||
builder.Services.AddSingleton(cacheOptions);
|
builder.Services.AddSingleton(cacheOptions);
|
||||||
builder.Services.AddSingleton(officialDocumentOptions);
|
builder.Services.AddSingleton(officialDocumentOptions);
|
||||||
builder.Services.AddSingleton(backgroundJobOptions);
|
builder.Services.AddSingleton(backgroundJobOptions);
|
||||||
|
builder.Services.AddSingleton(operationsOptions);
|
||||||
builder.Services.AddSingleton(rabbitMqOptions);
|
builder.Services.AddSingleton(rabbitMqOptions);
|
||||||
builder.Services.Configure<OfficialDocumentOptions>(
|
builder.Services.Configure<OfficialDocumentOptions>(
|
||||||
builder.Configuration.GetSection(OfficialDocumentOptions.SectionName));
|
builder.Configuration.GetSection(OfficialDocumentOptions.SectionName));
|
||||||
@@ -277,6 +296,8 @@ builder.Services.AddScoped<MakeupExamArrangementService>();
|
|||||||
builder.Services.AddScoped<MakeupExamAutoJobProcessor>();
|
builder.Services.AddScoped<MakeupExamAutoJobProcessor>();
|
||||||
builder.Services.AddSingleton<BackgroundJobTelemetry>();
|
builder.Services.AddSingleton<BackgroundJobTelemetry>();
|
||||||
builder.Services.AddScoped<BackgroundJobMonitoringService>();
|
builder.Services.AddScoped<BackgroundJobMonitoringService>();
|
||||||
|
builder.Services.AddScoped<OperationalHealthService>();
|
||||||
|
builder.Services.AddSingleton<DatabaseBackupService>();
|
||||||
builder.Services.AddSingleton<BackgroundJobRunner>();
|
builder.Services.AddSingleton<BackgroundJobRunner>();
|
||||||
if (backgroundJobOptions.UsesRabbitMq)
|
if (backgroundJobOptions.UsesRabbitMq)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -19,6 +19,14 @@
|
|||||||
"AnalyticsLocalExpirationSeconds": 30,
|
"AnalyticsLocalExpirationSeconds": 30,
|
||||||
"MaximumPayloadKilobytes": 2048
|
"MaximumPayloadKilobytes": 2048
|
||||||
},
|
},
|
||||||
|
"Operations": {
|
||||||
|
"BackupDirectory": "data/backups",
|
||||||
|
"BackupWarningHours": 24,
|
||||||
|
"ToolTimeoutMinutes": 30,
|
||||||
|
"MySqlDumpPath": "mysqldump",
|
||||||
|
"MySqlClientPath": "mysql",
|
||||||
|
"MySqlAdditionalArguments": []
|
||||||
|
},
|
||||||
"BackgroundJobs": {
|
"BackgroundJobs": {
|
||||||
"Transport": "InMemory",
|
"Transport": "InMemory",
|
||||||
"PollIntervalMilliseconds": 500,
|
"PollIntervalMilliseconds": 500,
|
||||||
|
|||||||
@@ -0,0 +1,252 @@
|
|||||||
|
using System.Reflection;
|
||||||
|
using Jiaowu.Api.Contracts;
|
||||||
|
using Jiaowu.Api.Controllers;
|
||||||
|
using Jiaowu.Api.Domain.Academic;
|
||||||
|
using Jiaowu.Api.Domain.Identity;
|
||||||
|
using Jiaowu.Api.Domain.System;
|
||||||
|
using Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||||
|
using Jiaowu.Api.Infrastructure.Caching;
|
||||||
|
using Jiaowu.Api.Infrastructure.Operations;
|
||||||
|
using Jiaowu.Api.Infrastructure.Persistence;
|
||||||
|
using Microsoft.AspNetCore.Authorization;
|
||||||
|
using Microsoft.AspNetCore.Hosting;
|
||||||
|
using Microsoft.AspNetCore.Mvc;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.Extensions.Configuration;
|
||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
using Microsoft.Extensions.FileProviders;
|
||||||
|
using Microsoft.Extensions.Hosting;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
|
namespace Jiaowu.Api.Tests;
|
||||||
|
|
||||||
|
public sealed class OperationsControllerTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void Controller_is_restricted_to_super_admin()
|
||||||
|
{
|
||||||
|
var authorize = typeof(OperationsController)
|
||||||
|
.GetCustomAttribute<AuthorizeAttribute>();
|
||||||
|
|
||||||
|
Assert.NotNull(authorize);
|
||||||
|
Assert.Equal(SystemRoles.SuperAdmin, authorize.Roles);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Audit_and_failed_job_queries_return_operational_records()
|
||||||
|
{
|
||||||
|
var root = CreateTemporaryRoot();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await using var fixture = await OperationsFixture.CreateAsync(root);
|
||||||
|
var term = new AcademicTerm
|
||||||
|
{
|
||||||
|
Code = "OPS-TERM",
|
||||||
|
Name = "运维测试学期",
|
||||||
|
AcademicYear = "2026-2027",
|
||||||
|
Season = TermSeason.Autumn,
|
||||||
|
StartDate = new DateOnly(2026, 9, 1),
|
||||||
|
EndDate = new DateOnly(2027, 1, 15)
|
||||||
|
};
|
||||||
|
var plan = new SchedulePlan
|
||||||
|
{
|
||||||
|
AcademicTerm = term,
|
||||||
|
Name = "恢复演练排课方案",
|
||||||
|
Version = "V1"
|
||||||
|
};
|
||||||
|
var job = new AutomaticScheduleJob
|
||||||
|
{
|
||||||
|
SchedulePlan = plan,
|
||||||
|
Status = AutomaticScheduleJobStatus.Failed,
|
||||||
|
ErrorMessage = "排课求解器未找到可行解。",
|
||||||
|
CompletedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
var outbox = BackgroundJobOutboxMessage.Create(
|
||||||
|
BackgroundJobKind.AutomaticSchedule,
|
||||||
|
job.Id);
|
||||||
|
outbox.State = BackgroundJobOutboxState.Completed;
|
||||||
|
outbox.ProcessingAttempts = 3;
|
||||||
|
fixture.Db.AuditLogs.Add(new AuditLog
|
||||||
|
{
|
||||||
|
UserName = "root",
|
||||||
|
Method = "POST",
|
||||||
|
Path = "/api/schedules",
|
||||||
|
StatusCode = 500,
|
||||||
|
IpAddress = "127.0.0.1"
|
||||||
|
});
|
||||||
|
fixture.Db.AddRange(term, plan, job, outbox);
|
||||||
|
await fixture.Db.SaveChangesAsync();
|
||||||
|
|
||||||
|
var auditAction = await fixture.Controller.GetAuditLogs(
|
||||||
|
cancellationToken: CancellationToken.None);
|
||||||
|
var auditResult = Assert.IsType<OkObjectResult>(auditAction.Result);
|
||||||
|
var auditPage = Assert.IsType<PagedResult<AuditLogItem>>(auditResult.Value);
|
||||||
|
Assert.Single(auditPage.Items);
|
||||||
|
Assert.Equal(500, auditPage.Items.Single().StatusCode);
|
||||||
|
|
||||||
|
var jobsAction = await fixture.Controller.GetFailedJobs(
|
||||||
|
cancellationToken: CancellationToken.None);
|
||||||
|
var jobsResult = Assert.IsType<OkObjectResult>(jobsAction.Result);
|
||||||
|
var jobsPage =
|
||||||
|
Assert.IsType<PagedResult<FailedBackgroundJobItem>>(jobsResult.Value);
|
||||||
|
var failed = Assert.Single(jobsPage.Items);
|
||||||
|
Assert.Equal("AutomaticSchedule", failed.Kind);
|
||||||
|
Assert.Equal(3, failed.ProcessingAttempts);
|
||||||
|
Assert.Contains("可行解", failed.ErrorMessage);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
DeleteTemporaryRoot(root);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Sqlite_backup_can_be_verified_in_an_isolated_restore_drill()
|
||||||
|
{
|
||||||
|
var root = CreateTemporaryRoot();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await using var fixture = await OperationsFixture.CreateAsync(root);
|
||||||
|
fixture.Db.AuditLogs.Add(new AuditLog
|
||||||
|
{
|
||||||
|
UserName = "backup-test",
|
||||||
|
Method = "POST",
|
||||||
|
Path = "/api/test",
|
||||||
|
StatusCode = 204
|
||||||
|
});
|
||||||
|
await fixture.Db.SaveChangesAsync();
|
||||||
|
|
||||||
|
var artifact = await fixture.Backups.CreateAsync(
|
||||||
|
"自动化测试",
|
||||||
|
CancellationToken.None);
|
||||||
|
var result = await fixture.Backups.RunRestoreDrillAsync(
|
||||||
|
artifact.Id,
|
||||||
|
CancellationToken.None);
|
||||||
|
var listed = await fixture.Backups.ListAsync(CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.True(File.Exists(Path.Combine(
|
||||||
|
root,
|
||||||
|
"backups",
|
||||||
|
artifact.FileName)));
|
||||||
|
Assert.Equal(64, artifact.Sha256.Length);
|
||||||
|
Assert.True(result.Succeeded, result.Detail);
|
||||||
|
Assert.True(result.TableCount > 0);
|
||||||
|
Assert.True(Assert.Single(listed).LastDrillSucceeded);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
DeleteTemporaryRoot(root);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string CreateTemporaryRoot()
|
||||||
|
{
|
||||||
|
var root = Path.Combine(
|
||||||
|
Path.GetTempPath(),
|
||||||
|
$"jiaowu-operations-{Guid.NewGuid():N}");
|
||||||
|
Directory.CreateDirectory(root);
|
||||||
|
return root;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void DeleteTemporaryRoot(string root)
|
||||||
|
{
|
||||||
|
var resolved = Path.GetFullPath(root);
|
||||||
|
var temporary = Path.GetFullPath(Path.GetTempPath());
|
||||||
|
Assert.StartsWith(temporary, resolved, StringComparison.OrdinalIgnoreCase);
|
||||||
|
if (Directory.Exists(resolved))
|
||||||
|
Directory.Delete(resolved, recursive: true);
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class OperationsFixture : IAsyncDisposable
|
||||||
|
{
|
||||||
|
private readonly ServiceProvider provider;
|
||||||
|
|
||||||
|
private OperationsFixture(
|
||||||
|
ServiceProvider provider,
|
||||||
|
AppDbContext db,
|
||||||
|
OperationsController controller,
|
||||||
|
DatabaseBackupService backups)
|
||||||
|
{
|
||||||
|
this.provider = provider;
|
||||||
|
Db = db;
|
||||||
|
Controller = controller;
|
||||||
|
Backups = backups;
|
||||||
|
}
|
||||||
|
|
||||||
|
public AppDbContext Db { get; }
|
||||||
|
public OperationsController Controller { get; }
|
||||||
|
public DatabaseBackupService Backups { get; }
|
||||||
|
|
||||||
|
public static async Task<OperationsFixture> CreateAsync(string root)
|
||||||
|
{
|
||||||
|
var databasePath = Path.Combine(root, "source.sqlite");
|
||||||
|
var configuration = new ConfigurationBuilder()
|
||||||
|
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||||
|
{
|
||||||
|
["ConnectionStrings:SQLite"] =
|
||||||
|
$"Data Source={databasePath};Pooling=False"
|
||||||
|
})
|
||||||
|
.Build();
|
||||||
|
var databaseOptions = new DatabaseOptions { Provider = "SQLite" };
|
||||||
|
var cacheOptions = new AppCacheOptions();
|
||||||
|
var operationsOptions = new OperationsOptions
|
||||||
|
{
|
||||||
|
BackupDirectory = "backups"
|
||||||
|
};
|
||||||
|
var dbContextOptions = new DbContextOptionsBuilder<AppDbContext>()
|
||||||
|
.UseSqlite(configuration.GetConnectionString("SQLite"))
|
||||||
|
.Options;
|
||||||
|
var db = new AppDbContext(dbContextOptions);
|
||||||
|
await db.Database.EnsureCreatedAsync();
|
||||||
|
|
||||||
|
var services = new ServiceCollection()
|
||||||
|
.AddLogging()
|
||||||
|
.BuildServiceProvider();
|
||||||
|
var logger = services.GetRequiredService<
|
||||||
|
ILogger<DatabaseBackupService>>();
|
||||||
|
var environment = new TestHostEnvironment
|
||||||
|
{
|
||||||
|
ContentRootPath = root
|
||||||
|
};
|
||||||
|
var transport = new InMemoryBackgroundJobTransport();
|
||||||
|
var monitoring = new BackgroundJobMonitoringService(db);
|
||||||
|
var health = new OperationalHealthService(
|
||||||
|
db,
|
||||||
|
services,
|
||||||
|
transport,
|
||||||
|
monitoring,
|
||||||
|
databaseOptions,
|
||||||
|
cacheOptions,
|
||||||
|
configuration);
|
||||||
|
var backups = new DatabaseBackupService(
|
||||||
|
databaseOptions,
|
||||||
|
operationsOptions,
|
||||||
|
configuration,
|
||||||
|
environment,
|
||||||
|
logger);
|
||||||
|
var controller = new OperationsController(
|
||||||
|
db,
|
||||||
|
health,
|
||||||
|
backups,
|
||||||
|
operationsOptions);
|
||||||
|
return new OperationsFixture(services, db, controller, backups);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async ValueTask DisposeAsync()
|
||||||
|
{
|
||||||
|
await Db.DisposeAsync();
|
||||||
|
await provider.DisposeAsync();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class TestHostEnvironment : IWebHostEnvironment
|
||||||
|
{
|
||||||
|
public string ApplicationName { get; set; } = "Jiaowu.Api.Tests";
|
||||||
|
public IFileProvider WebRootFileProvider { get; set; } = new NullFileProvider();
|
||||||
|
public string WebRootPath { get; set; } = "";
|
||||||
|
public string EnvironmentName { get; set; } = Environments.Development;
|
||||||
|
public string ContentRootPath { get; set; } = "";
|
||||||
|
public IFileProvider ContentRootFileProvider { get; set; } =
|
||||||
|
new NullFileProvider();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -97,6 +97,7 @@ const navigationGroups = computed<NavigationGroup[]>(() => [
|
|||||||
{ path: '/students', label: '学生档案' },
|
{ path: '/students', label: '学生档案' },
|
||||||
),
|
),
|
||||||
...whenVisible(auth.isSuperAdmin, { path: '/users', label: '用户与权限' }),
|
...whenVisible(auth.isSuperAdmin, { path: '/users', label: '用户与权限' }),
|
||||||
|
...whenVisible(auth.isSuperAdmin, { path: '/operations', label: '运维与审计' }),
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -291,6 +291,12 @@ const router = createRouter({
|
|||||||
component: () => import('../views/UsersView.vue'),
|
component: () => import('../views/UsersView.vue'),
|
||||||
meta: { roles: ['SuperAdmin'] },
|
meta: { roles: ['SuperAdmin'] },
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
path: 'operations',
|
||||||
|
name: 'operations',
|
||||||
|
component: () => import('../views/OperationsConsoleView.vue'),
|
||||||
|
meta: { roles: ['SuperAdmin'] },
|
||||||
|
},
|
||||||
{
|
{
|
||||||
path: 'evaluations',
|
path: 'evaluations',
|
||||||
name: 'evaluations',
|
name: 'evaluations',
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user