第二阶段优化已完成,重点是提高后台任务吞吐并降低 Outbox 数据库开销。
RabbitMQ 默认从单一回调通道提升为三类任务独立并行,跨类型并发能力由 1 提升到 3;每类还能独立配置 1–16 个消费者。[RabbitMqBackgroundJobs.cs (line 143)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/RabbitMqBackgroundJobs.cs:143) InMemory 开发模式同步改为按任务类型隔离队列,避免某类长任务堵塞其他任务。 租约恢复检查从“每发布一条执行一次”改为默认每 60 秒维护一次。 启动恢复改为数据库 NOT EXISTS 查询,不再把全部历史 Outbox 加载进内存。[BackgroundJobOutboxPublisher.cs (line 9)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/BackgroundJobOutboxPublisher.cs:9) 完成消息默认保留 14 天,之后按每批 500 条清理,并增加对应组合索引。 /health/messaging 现在返回各状态积压量、过期租约和最老任务等待时间。[BackgroundJobMonitoringService.cs (line 16)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/BackgroundJobMonitoringService.cs:16) 新增 Jiaowu.BackgroundJobs 运行时指标,覆盖发布量、处理量、发布耗时、处理耗时和清理量。[BackgroundJobTelemetry.cs (line 9)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/BackgroundJobTelemetry.cs:9) 配置、Compose 和调优建议已更新。[README.md (line 257)](E:/jiaowu/README.md:257)
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
using System.Diagnostics;
|
||||
using Jiaowu.Api.Domain.Academic;
|
||||
using Jiaowu.Api.Domain.System;
|
||||
using Jiaowu.Api.Infrastructure.Persistence;
|
||||
@@ -9,17 +10,26 @@ public sealed class BackgroundJobOutboxPublisher(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IBackgroundJobTransport transport,
|
||||
BackgroundJobOptions options,
|
||||
BackgroundJobTelemetry telemetry,
|
||||
ILogger<BackgroundJobOutboxPublisher> logger) : BackgroundService
|
||||
{
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
await PrepareForStartupAsync(stoppingToken);
|
||||
var nextMaintenanceAt = DateTime.MinValue;
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await RecoverExpiredProcessingAsync(stoppingToken);
|
||||
if (DateTime.UtcNow >= nextMaintenanceAt)
|
||||
{
|
||||
await RecoverExpiredProcessingAsync(stoppingToken);
|
||||
await CleanupCompletedAsync(stoppingToken);
|
||||
nextMaintenanceAt = DateTime.UtcNow.AddSeconds(
|
||||
options.MaintenanceIntervalSeconds);
|
||||
}
|
||||
|
||||
var claimed = await ClaimNextAsync(stoppingToken);
|
||||
if (claimed is null)
|
||||
{
|
||||
@@ -62,51 +72,107 @@ public sealed class BackgroundJobOutboxPublisher(
|
||||
await AddMissingOutboxMessagesAsync(db, cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task AddMissingOutboxMessagesAsync(
|
||||
private async Task AddMissingOutboxMessagesAsync(
|
||||
AppDbContext db,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var existing = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||
.Select(x => new { x.JobKind, x.JobId })
|
||||
.ToListAsync(cancellationToken);
|
||||
var existingKeys = existing
|
||||
.Select(x => (x.JobKind, x.JobId))
|
||||
.ToHashSet();
|
||||
|
||||
var automaticJobs = await db.AutomaticScheduleJobs.AsNoTracking()
|
||||
.Where(x => x.Status == AutomaticScheduleJobStatus.Queued ||
|
||||
x.Status == AutomaticScheduleJobStatus.Running)
|
||||
.Where(x =>
|
||||
(x.Status == AutomaticScheduleJobStatus.Queued ||
|
||||
x.Status == AutomaticScheduleJobStatus.Running) &&
|
||||
!db.BackgroundJobOutboxMessages.Any(message =>
|
||||
message.JobKind == BackgroundJobKind.AutomaticSchedule &&
|
||||
message.JobId == x.Id))
|
||||
.Select(x => x.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
var publishJobs = await db.SchedulePublishJobs.AsNoTracking()
|
||||
.Where(x => x.Status == SchedulePublishJobStatus.Queued ||
|
||||
x.Status == SchedulePublishJobStatus.Running)
|
||||
.Where(x =>
|
||||
(x.Status == SchedulePublishJobStatus.Queued ||
|
||||
x.Status == SchedulePublishJobStatus.Running) &&
|
||||
!db.BackgroundJobOutboxMessages.Any(message =>
|
||||
message.JobKind == BackgroundJobKind.SchedulePublish &&
|
||||
message.JobId == x.Id))
|
||||
.Select(x => x.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
var makeupJobs = await db.MakeupExamAutoJobs.AsNoTracking()
|
||||
.Where(x => x.Status == MakeupExamAutoJobStatus.Queued ||
|
||||
x.Status == MakeupExamAutoJobStatus.Running)
|
||||
.Where(x =>
|
||||
(x.Status == MakeupExamAutoJobStatus.Queued ||
|
||||
x.Status == MakeupExamAutoJobStatus.Running) &&
|
||||
!db.BackgroundJobOutboxMessages.Any(message =>
|
||||
message.JobKind == BackgroundJobKind.MakeupExamAuto &&
|
||||
message.JobId == x.Id))
|
||||
.Select(x => x.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
AddMissing(BackgroundJobKind.AutomaticSchedule, automaticJobs);
|
||||
AddMissing(BackgroundJobKind.SchedulePublish, publishJobs);
|
||||
AddMissing(BackgroundJobKind.MakeupExamAuto, makeupJobs);
|
||||
|
||||
if (db.ChangeTracker.HasChanges())
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
|
||||
void AddMissing(BackgroundJobKind kind, IEnumerable<Guid> jobIds)
|
||||
var missingKeys = automaticJobs
|
||||
.Select(id => (BackgroundJobKind.AutomaticSchedule, id))
|
||||
.Concat(publishJobs.Select(id =>
|
||||
(BackgroundJobKind.SchedulePublish, id)))
|
||||
.Concat(makeupJobs.Select(id =>
|
||||
(BackgroundJobKind.MakeupExamAuto, id)))
|
||||
.ToList();
|
||||
foreach (var (kind, jobId) in missingKeys)
|
||||
{
|
||||
foreach (var jobId in jobIds)
|
||||
{
|
||||
if (existingKeys.Add((kind, jobId)))
|
||||
{
|
||||
db.BackgroundJobOutboxMessages.Add(
|
||||
BackgroundJobOutboxMessage.Create(kind, jobId));
|
||||
}
|
||||
}
|
||||
db.BackgroundJobOutboxMessages.Add(
|
||||
BackgroundJobOutboxMessage.Create(kind, jobId));
|
||||
}
|
||||
|
||||
if (!db.ChangeTracker.HasChanges())
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
catch (DbUpdateException)
|
||||
{
|
||||
db.ChangeTracker.Clear();
|
||||
foreach (var (kind, jobId) in missingKeys)
|
||||
{
|
||||
var recovered = await db.BackgroundJobOutboxMessages
|
||||
.AsNoTracking()
|
||||
.AnyAsync(
|
||||
message => message.JobKind == kind &&
|
||||
message.JobId == jobId,
|
||||
cancellationToken);
|
||||
if (!recovered)
|
||||
throw;
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Another application instance recovered {Count} missing outbox messages first.",
|
||||
missingKeys.Count);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CleanupCompletedAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
var cutoff = DateTime.UtcNow.AddDays(-options.CompletedRetentionDays);
|
||||
var completedIds = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||
.Where(x =>
|
||||
x.State == BackgroundJobOutboxState.Completed &&
|
||||
x.CompletedAt != null &&
|
||||
x.CompletedAt < cutoff)
|
||||
.OrderBy(x => x.CompletedAt)
|
||||
.Select(x => x.Id)
|
||||
.Take(options.CleanupBatchSize)
|
||||
.ToListAsync(cancellationToken);
|
||||
if (completedIds.Count == 0)
|
||||
return;
|
||||
|
||||
foreach (var id in completedIds)
|
||||
{
|
||||
db.BackgroundJobOutboxMessages.Remove(
|
||||
new BackgroundJobOutboxMessage { Id = id });
|
||||
}
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
telemetry.RecordCleaned(completedIds.Count);
|
||||
logger.LogInformation(
|
||||
"Removed {Count} completed background job outbox messages older than {Cutoff}.",
|
||||
completedIds.Count,
|
||||
cutoff);
|
||||
}
|
||||
|
||||
private async Task RecoverExpiredProcessingAsync(
|
||||
@@ -175,6 +241,7 @@ public sealed class BackgroundJobOutboxPublisher(
|
||||
Guid token,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var startedAt = Stopwatch.GetTimestamp();
|
||||
try
|
||||
{
|
||||
await transport.PublishAsync(message, cancellationToken);
|
||||
@@ -193,6 +260,11 @@ public sealed class BackgroundJobOutboxPublisher(
|
||||
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null),
|
||||
cancellationToken);
|
||||
telemetry.RecordPublish(
|
||||
message.JobKind,
|
||||
transport.IsDurable,
|
||||
"published",
|
||||
Stopwatch.GetElapsedTime(startedAt));
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
@@ -213,6 +285,11 @@ public sealed class BackgroundJobOutboxPublisher(
|
||||
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null)
|
||||
.SetProperty(x => x.LastError, messageText),
|
||||
cancellationToken);
|
||||
telemetry.RecordPublish(
|
||||
message.JobKind,
|
||||
transport.IsDurable,
|
||||
"failed",
|
||||
Stopwatch.GetElapsedTime(startedAt));
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user