主要能力: SuperAdmin 专用入口:组织与权限 → 运维与审计。 操作日志分页查询,支持时间、账号、路径、方法和状态码筛选。 汇总自动排课、课表发布、补考安排三类失败后台任务。 实时检查数据库、缓存、任务通道及积压状态。 聚合 5xx、失败/重试任务、健康探针和备份时效告警。 SQLite 在线备份;MySQL 调用原生客户端备份。 SHA-256 校验及隔离数据库恢复演练,不覆盖业务库。 MySQL 强制使用独立运维连接,容器增加持久化备份卷与数据库客户端。
585 lines
21 KiB
C#
585 lines
21 KiB
C#
using System.Diagnostics;
|
|
using System.Security.Cryptography;
|
|
using System.Text.Json;
|
|
using Jiaowu.Api.Infrastructure.Persistence;
|
|
using Microsoft.Data.Sqlite;
|
|
using MySql.Data.MySqlClient;
|
|
|
|
namespace Jiaowu.Api.Infrastructure.Operations;
|
|
|
|
public sealed record BackupArtifact(
|
|
string Id,
|
|
string FileName,
|
|
string Provider,
|
|
DateTime CreatedAt,
|
|
long SizeBytes,
|
|
string Sha256,
|
|
string? Note,
|
|
DateTime? LastDrillAt,
|
|
bool? LastDrillSucceeded,
|
|
string? LastDrillDetail,
|
|
long? LastDrillDurationMilliseconds);
|
|
|
|
public sealed record RestoreDrillResult(
|
|
string BackupId,
|
|
bool Succeeded,
|
|
DateTime CompletedAt,
|
|
string Detail,
|
|
long DurationMilliseconds,
|
|
int? TableCount);
|
|
|
|
public sealed class DatabaseBackupService(
|
|
DatabaseOptions databaseOptions,
|
|
OperationsOptions options,
|
|
IConfiguration configuration,
|
|
IHostEnvironment environment,
|
|
ILogger<DatabaseBackupService> logger)
|
|
{
|
|
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
|
{
|
|
WriteIndented = true
|
|
};
|
|
|
|
private readonly SemaphoreSlim operationLock = new(1, 1);
|
|
private readonly string backupDirectory = ResolveBackupDirectory(
|
|
options.BackupDirectory,
|
|
environment.ContentRootPath);
|
|
|
|
public async Task<IReadOnlyCollection<BackupArtifact>> ListAsync(
|
|
CancellationToken cancellationToken)
|
|
{
|
|
Directory.CreateDirectory(backupDirectory);
|
|
var items = new List<BackupArtifact>();
|
|
foreach (var metadataPath in Directory.EnumerateFiles(
|
|
backupDirectory,
|
|
"*.metadata.json",
|
|
SearchOption.TopDirectoryOnly))
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
try
|
|
{
|
|
await using var stream = File.OpenRead(metadataPath);
|
|
var artifact = await JsonSerializer.DeserializeAsync<BackupArtifact>(
|
|
stream,
|
|
JsonOptions,
|
|
cancellationToken);
|
|
if (artifact is not null &&
|
|
File.Exists(Path.Combine(backupDirectory, artifact.FileName)))
|
|
{
|
|
items.Add(artifact);
|
|
}
|
|
}
|
|
catch (Exception exception) when (
|
|
exception is IOException or UnauthorizedAccessException or JsonException)
|
|
{
|
|
logger.LogWarning(
|
|
exception,
|
|
"Unable to read backup metadata {MetadataFile}.",
|
|
Path.GetFileName(metadataPath));
|
|
}
|
|
}
|
|
|
|
return items
|
|
.OrderByDescending(x => x.CreatedAt)
|
|
.ToArray();
|
|
}
|
|
|
|
public async Task<BackupArtifact> CreateAsync(
|
|
string? note,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
await operationLock.WaitAsync(cancellationToken);
|
|
try
|
|
{
|
|
Directory.CreateDirectory(backupDirectory);
|
|
var createdAt = DateTime.UtcNow;
|
|
var id = $"{createdAt:yyyyMMddHHmmss}-{Guid.NewGuid():N}"[..29];
|
|
var provider = NormalizeProvider(databaseOptions.Provider);
|
|
var extension = provider == "SQLite" ? ".sqlite" : ".sql";
|
|
var fileName = $"jiaowu-{id}{extension}";
|
|
var backupPath = Path.Combine(backupDirectory, fileName);
|
|
|
|
try
|
|
{
|
|
if (provider == "SQLite")
|
|
await CreateSqliteBackupAsync(backupPath, cancellationToken);
|
|
else
|
|
await CreateMySqlBackupAsync(backupPath, cancellationToken);
|
|
|
|
var fileInfo = new FileInfo(backupPath);
|
|
var artifact = new BackupArtifact(
|
|
id,
|
|
fileName,
|
|
provider,
|
|
createdAt,
|
|
fileInfo.Length,
|
|
await ComputeHashAsync(backupPath, cancellationToken),
|
|
NormalizeNote(note),
|
|
null,
|
|
null,
|
|
null,
|
|
null);
|
|
await WriteMetadataAsync(artifact, cancellationToken);
|
|
return artifact;
|
|
}
|
|
catch
|
|
{
|
|
if (File.Exists(backupPath))
|
|
File.Delete(backupPath);
|
|
throw;
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
operationLock.Release();
|
|
}
|
|
}
|
|
|
|
public async Task<RestoreDrillResult> RunRestoreDrillAsync(
|
|
string backupId,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
await operationLock.WaitAsync(cancellationToken);
|
|
try
|
|
{
|
|
var artifact = (await ListAsync(cancellationToken))
|
|
.SingleOrDefault(x => x.Id.Equals(backupId, StringComparison.Ordinal));
|
|
if (artifact is null)
|
|
throw new FileNotFoundException("未找到指定备份。");
|
|
|
|
var backupPath = Path.Combine(backupDirectory, artifact.FileName);
|
|
var actualHash = await ComputeHashAsync(backupPath, cancellationToken);
|
|
if (!CryptographicOperations.FixedTimeEquals(
|
|
Convert.FromHexString(artifact.Sha256),
|
|
Convert.FromHexString(actualHash)))
|
|
{
|
|
var damaged = await CompleteDrillAsync(
|
|
artifact,
|
|
false,
|
|
"备份文件校验和不一致,恢复演练已中止。",
|
|
0,
|
|
null,
|
|
cancellationToken);
|
|
return damaged;
|
|
}
|
|
|
|
var stopwatch = Stopwatch.StartNew();
|
|
RestoreDrillResult result;
|
|
try
|
|
{
|
|
var tableCount = artifact.Provider == "SQLite"
|
|
? await DrillSqliteAsync(backupPath, cancellationToken)
|
|
: await DrillMySqlAsync(backupPath, cancellationToken);
|
|
stopwatch.Stop();
|
|
result = await CompleteDrillAsync(
|
|
artifact,
|
|
true,
|
|
$"已在隔离数据库完成恢复并通过完整性检查,共发现 {tableCount} 张业务表。",
|
|
stopwatch.ElapsedMilliseconds,
|
|
tableCount,
|
|
cancellationToken);
|
|
}
|
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
|
{
|
|
stopwatch.Stop();
|
|
logger.LogError(
|
|
exception,
|
|
"Restore drill failed for backup {BackupId}.",
|
|
artifact.Id);
|
|
result = await CompleteDrillAsync(
|
|
artifact,
|
|
false,
|
|
$"恢复演练失败:{SafeMessage(exception)}",
|
|
stopwatch.ElapsedMilliseconds,
|
|
null,
|
|
cancellationToken);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
finally
|
|
{
|
|
operationLock.Release();
|
|
}
|
|
}
|
|
|
|
private async Task CreateSqliteBackupAsync(
|
|
string backupPath,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var sourceBuilder = new SqliteConnectionStringBuilder(
|
|
configuration.GetConnectionString("SQLite")
|
|
?? throw new InvalidOperationException("缺少 ConnectionStrings:SQLite。"));
|
|
if (!Path.IsPathRooted(sourceBuilder.DataSource))
|
|
{
|
|
sourceBuilder.DataSource = Path.GetFullPath(
|
|
sourceBuilder.DataSource,
|
|
environment.ContentRootPath);
|
|
}
|
|
|
|
var destinationBuilder = new SqliteConnectionStringBuilder
|
|
{
|
|
DataSource = backupPath,
|
|
Mode = SqliteOpenMode.ReadWriteCreate,
|
|
Pooling = false
|
|
};
|
|
await using var source = new SqliteConnection(sourceBuilder.ConnectionString);
|
|
await using var destination = new SqliteConnection(destinationBuilder.ConnectionString);
|
|
await source.OpenAsync(cancellationToken);
|
|
await destination.OpenAsync(cancellationToken);
|
|
source.BackupDatabase(destination);
|
|
}
|
|
|
|
private async Task CreateMySqlBackupAsync(
|
|
string backupPath,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var connection = GetMySqlConnectionBuilder();
|
|
var arguments = new List<string>
|
|
{
|
|
"--protocol=tcp",
|
|
$"--host={connection.Server}",
|
|
$"--port={connection.Port}",
|
|
$"--user={connection.UserID}",
|
|
"--single-transaction",
|
|
"--quick",
|
|
"--routines",
|
|
"--triggers",
|
|
"--events",
|
|
"--hex-blob",
|
|
"--default-character-set=utf8mb4"
|
|
};
|
|
arguments.AddRange(options.MySqlAdditionalArguments);
|
|
arguments.Add(connection.Database);
|
|
|
|
await using var output = new FileStream(
|
|
backupPath,
|
|
FileMode.CreateNew,
|
|
FileAccess.Write,
|
|
FileShare.None,
|
|
81920,
|
|
FileOptions.Asynchronous);
|
|
await RunToolAsync(
|
|
options.MySqlDumpPath,
|
|
arguments,
|
|
connection.Password,
|
|
standardInput: null,
|
|
standardOutput: output,
|
|
cancellationToken);
|
|
}
|
|
|
|
private static async Task<int> DrillSqliteAsync(
|
|
string backupPath,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var drillPath = Path.Combine(
|
|
Path.GetDirectoryName(backupPath)!,
|
|
$".restore-drill-{Guid.NewGuid():N}.sqlite");
|
|
try
|
|
{
|
|
var sourceBuilder = new SqliteConnectionStringBuilder
|
|
{
|
|
DataSource = backupPath,
|
|
Mode = SqliteOpenMode.ReadOnly,
|
|
Pooling = false
|
|
};
|
|
var drillBuilder = new SqliteConnectionStringBuilder
|
|
{
|
|
DataSource = drillPath,
|
|
Mode = SqliteOpenMode.ReadWriteCreate,
|
|
Pooling = false
|
|
};
|
|
await using var source = new SqliteConnection(sourceBuilder.ConnectionString);
|
|
await using var drill = new SqliteConnection(drillBuilder.ConnectionString);
|
|
await source.OpenAsync(cancellationToken);
|
|
await drill.OpenAsync(cancellationToken);
|
|
source.BackupDatabase(drill);
|
|
|
|
await using var integrity = drill.CreateCommand();
|
|
integrity.CommandText = "PRAGMA integrity_check;";
|
|
var integrityResult = Convert.ToString(
|
|
await integrity.ExecuteScalarAsync(cancellationToken));
|
|
if (!string.Equals(integrityResult, "ok", StringComparison.OrdinalIgnoreCase))
|
|
throw new InvalidDataException($"SQLite 完整性检查返回 {integrityResult ?? "空结果"}。");
|
|
|
|
await using var tables = drill.CreateCommand();
|
|
tables.CommandText =
|
|
"SELECT COUNT(*) FROM sqlite_master " +
|
|
"WHERE type = 'table' AND name NOT LIKE 'sqlite_%';";
|
|
return Convert.ToInt32(await tables.ExecuteScalarAsync(cancellationToken));
|
|
}
|
|
finally
|
|
{
|
|
if (File.Exists(drillPath))
|
|
File.Delete(drillPath);
|
|
if (File.Exists($"{drillPath}-shm"))
|
|
File.Delete($"{drillPath}-shm");
|
|
if (File.Exists($"{drillPath}-wal"))
|
|
File.Delete($"{drillPath}-wal");
|
|
}
|
|
}
|
|
|
|
private async Task<int> DrillMySqlAsync(
|
|
string backupPath,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var connection = GetMySqlConnectionBuilder();
|
|
var drillDatabase = $"jiaowu_restore_drill_{DateTime.UtcNow:yyyyMMddHHmmss}_" +
|
|
Guid.NewGuid().ToString("N")[..8];
|
|
var adminBuilder = new MySqlConnectionStringBuilder(connection.ConnectionString)
|
|
{
|
|
Database = ""
|
|
};
|
|
|
|
await using var admin = new MySqlConnection(adminBuilder.ConnectionString);
|
|
await admin.OpenAsync(cancellationToken);
|
|
try
|
|
{
|
|
await using (var create = admin.CreateCommand())
|
|
{
|
|
create.CommandText =
|
|
$"CREATE DATABASE `{drillDatabase}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;";
|
|
await create.ExecuteNonQueryAsync(cancellationToken);
|
|
}
|
|
|
|
var arguments = new List<string>
|
|
{
|
|
"--protocol=tcp",
|
|
$"--host={connection.Server}",
|
|
$"--port={connection.Port}",
|
|
$"--user={connection.UserID}",
|
|
"--default-character-set=utf8mb4"
|
|
};
|
|
arguments.AddRange(options.MySqlAdditionalArguments);
|
|
arguments.Add($"--database={drillDatabase}");
|
|
await using (var input = new FileStream(
|
|
backupPath,
|
|
FileMode.Open,
|
|
FileAccess.Read,
|
|
FileShare.Read,
|
|
81920,
|
|
FileOptions.Asynchronous))
|
|
{
|
|
await RunToolAsync(
|
|
options.MySqlClientPath,
|
|
arguments,
|
|
connection.Password,
|
|
input,
|
|
standardOutput: null,
|
|
cancellationToken);
|
|
}
|
|
|
|
await using var count = admin.CreateCommand();
|
|
count.CommandText =
|
|
"SELECT COUNT(*) FROM information_schema.tables " +
|
|
"WHERE table_schema = @schema AND table_type = 'BASE TABLE';";
|
|
count.Parameters.AddWithValue("@schema", drillDatabase);
|
|
return Convert.ToInt32(await count.ExecuteScalarAsync(cancellationToken));
|
|
}
|
|
finally
|
|
{
|
|
try
|
|
{
|
|
await using var drop = admin.CreateCommand();
|
|
drop.CommandText = $"DROP DATABASE IF EXISTS `{drillDatabase}`;";
|
|
await drop.ExecuteNonQueryAsync(CancellationToken.None);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
logger.LogCritical(
|
|
exception,
|
|
"Unable to remove isolated restore drill database {DatabaseName}.",
|
|
drillDatabase);
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task RunToolAsync(
|
|
string executable,
|
|
IReadOnlyCollection<string> arguments,
|
|
string password,
|
|
Stream? standardInput,
|
|
Stream? standardOutput,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var startInfo = new ProcessStartInfo
|
|
{
|
|
FileName = executable,
|
|
UseShellExecute = false,
|
|
RedirectStandardError = true,
|
|
RedirectStandardInput = standardInput is not null,
|
|
RedirectStandardOutput = standardOutput is not null,
|
|
CreateNoWindow = true
|
|
};
|
|
foreach (var argument in arguments)
|
|
startInfo.ArgumentList.Add(argument);
|
|
if (!string.IsNullOrEmpty(password))
|
|
startInfo.Environment["MYSQL_PWD"] = password;
|
|
|
|
using var process = Process.Start(startInfo)
|
|
?? throw new InvalidOperationException($"无法启动数据库工具 {executable}。");
|
|
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
|
timeout.CancelAfter(TimeSpan.FromMinutes(options.ToolTimeoutMinutes));
|
|
var errorTask = process.StandardError.ReadToEndAsync(timeout.Token);
|
|
var inputTask = standardInput is null
|
|
? Task.CompletedTask
|
|
: CopyInputAsync(standardInput, process.StandardInput.BaseStream, timeout.Token);
|
|
var outputTask = standardOutput is null
|
|
? Task.CompletedTask
|
|
: process.StandardOutput.BaseStream.CopyToAsync(
|
|
standardOutput,
|
|
timeout.Token);
|
|
|
|
try
|
|
{
|
|
await Task.WhenAll(
|
|
process.WaitForExitAsync(timeout.Token),
|
|
inputTask,
|
|
outputTask);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
if (!process.HasExited)
|
|
process.Kill(entireProcessTree: true);
|
|
throw;
|
|
}
|
|
|
|
var error = await errorTask;
|
|
if (process.ExitCode != 0)
|
|
throw new InvalidOperationException(
|
|
$"数据库工具执行失败(退出码 {process.ExitCode}):{TrimToolError(error)}");
|
|
}
|
|
|
|
private static async Task CopyInputAsync(
|
|
Stream input,
|
|
Stream processInput,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
await input.CopyToAsync(processInput, cancellationToken);
|
|
await processInput.FlushAsync(cancellationToken);
|
|
processInput.Close();
|
|
}
|
|
|
|
private MySqlConnectionStringBuilder GetMySqlConnectionBuilder()
|
|
{
|
|
var value = configuration.GetConnectionString("OperationsMySql");
|
|
if (string.IsNullOrWhiteSpace(value))
|
|
{
|
|
throw new InvalidOperationException(
|
|
"MySQL 备份与恢复演练必须配置独立的 " +
|
|
"ConnectionStrings:OperationsMySql 运维账号,不能复用日常业务账号。");
|
|
}
|
|
var builder = new MySqlConnectionStringBuilder(value);
|
|
if (string.IsNullOrWhiteSpace(builder.Database))
|
|
throw new InvalidOperationException(
|
|
"OperationsMySql 连接字符串未指定业务数据库名称。");
|
|
return builder;
|
|
}
|
|
|
|
private async Task<RestoreDrillResult> CompleteDrillAsync(
|
|
BackupArtifact artifact,
|
|
bool succeeded,
|
|
string detail,
|
|
long durationMilliseconds,
|
|
int? tableCount,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var completedAt = DateTime.UtcNow;
|
|
var updated = artifact with
|
|
{
|
|
LastDrillAt = completedAt,
|
|
LastDrillSucceeded = succeeded,
|
|
LastDrillDetail = detail,
|
|
LastDrillDurationMilliseconds = durationMilliseconds
|
|
};
|
|
await WriteMetadataAsync(updated, cancellationToken);
|
|
return new RestoreDrillResult(
|
|
artifact.Id,
|
|
succeeded,
|
|
completedAt,
|
|
detail,
|
|
durationMilliseconds,
|
|
tableCount);
|
|
}
|
|
|
|
private async Task WriteMetadataAsync(
|
|
BackupArtifact artifact,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var metadataPath = Path.Combine(
|
|
backupDirectory,
|
|
$"{artifact.Id}.metadata.json");
|
|
var temporaryPath = $"{metadataPath}.{Guid.NewGuid():N}.tmp";
|
|
try
|
|
{
|
|
await using (var stream = new FileStream(
|
|
temporaryPath,
|
|
FileMode.CreateNew,
|
|
FileAccess.Write,
|
|
FileShare.None,
|
|
16384,
|
|
FileOptions.Asynchronous))
|
|
{
|
|
await JsonSerializer.SerializeAsync(
|
|
stream,
|
|
artifact,
|
|
JsonOptions,
|
|
cancellationToken);
|
|
}
|
|
File.Move(temporaryPath, metadataPath, overwrite: true);
|
|
}
|
|
finally
|
|
{
|
|
if (File.Exists(temporaryPath))
|
|
File.Delete(temporaryPath);
|
|
}
|
|
}
|
|
|
|
private static async Task<string> ComputeHashAsync(
|
|
string path,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
await using var stream = File.OpenRead(path);
|
|
return Convert.ToHexString(
|
|
await SHA256.HashDataAsync(stream, cancellationToken));
|
|
}
|
|
|
|
private static string ResolveBackupDirectory(
|
|
string configuredPath,
|
|
string contentRoot)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(configuredPath))
|
|
throw new InvalidOperationException("Operations:BackupDirectory 不能为空。");
|
|
return Path.IsPathRooted(configuredPath)
|
|
? Path.GetFullPath(configuredPath)
|
|
: Path.GetFullPath(configuredPath, contentRoot);
|
|
}
|
|
|
|
private static string NormalizeProvider(string provider) =>
|
|
provider.Equals("SQLite", StringComparison.OrdinalIgnoreCase)
|
|
? "SQLite"
|
|
: provider.Equals("MySql", StringComparison.OrdinalIgnoreCase)
|
|
? "MySql"
|
|
: throw new InvalidOperationException($"不支持数据库 Provider '{provider}'。");
|
|
|
|
private static string? NormalizeNote(string? note)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(note)) return null;
|
|
var trimmed = note.Trim();
|
|
return trimmed.Length <= 200 ? trimmed : trimmed[..200];
|
|
}
|
|
|
|
private static string SafeMessage(Exception exception)
|
|
{
|
|
var message = exception.GetBaseException().Message;
|
|
return message.Length <= 500 ? message : message[..500];
|
|
}
|
|
|
|
private static string TrimToolError(string error)
|
|
{
|
|
var trimmed = error.Trim();
|
|
if (trimmed.Length == 0) return "未返回错误详情。";
|
|
return trimmed.Length <= 500 ? trimmed : trimmed[..500];
|
|
}
|
|
}
|