第二阶段优化已完成,重点是提高后台任务吞吐并降低 Outbox 数据库开销。

RabbitMQ 默认从单一回调通道提升为三类任务独立并行,跨类型并发能力由 1 提升到 3;每类还能独立配置 1–16 个消费者。[RabbitMqBackgroundJobs.cs (line 143)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/RabbitMqBackgroundJobs.cs:143)
InMemory 开发模式同步改为按任务类型隔离队列,避免某类长任务堵塞其他任务。
租约恢复检查从“每发布一条执行一次”改为默认每 60 秒维护一次。
启动恢复改为数据库 NOT EXISTS 查询,不再把全部历史 Outbox 加载进内存。[BackgroundJobOutboxPublisher.cs (line 9)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/BackgroundJobOutboxPublisher.cs:9)
完成消息默认保留 14 天,之后按每批 500 条清理,并增加对应组合索引。
/health/messaging 现在返回各状态积压量、过期租约和最老任务等待时间。[BackgroundJobMonitoringService.cs (line 16)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/BackgroundJobMonitoringService.cs:16)
新增 Jiaowu.BackgroundJobs 运行时指标,覆盖发布量、处理量、发布耗时、处理耗时和清理量。[BackgroundJobTelemetry.cs (line 9)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/BackgroundJobTelemetry.cs:9)
配置、Compose 和调优建议已更新。[README.md (line 257)](E:/jiaowu/README.md:257)
This commit is contained in:
biss committed 2026-07-26 21:05:18 +08:00
1 parent 6300645120
commit 034f1eacb8
21 files changed
+579 -135

No files matched your search

@@ -0,0 +1,56 @@
using Jiaowu.Api.Domain.System;
using Jiaowu.Api.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
public sealed record BackgroundJobBacklogSnapshot(
int Pending,
int Publishing,
int Published,
int Processing,
int ExpiredLeases,
DateTime? OldestUnfinishedAt,
double? OldestUnfinishedAgeSeconds);
public sealed class BackgroundJobMonitoringService(AppDbContext db)
{
public async Task<BackgroundJobBacklogSnapshot> GetSnapshotAsync(
CancellationToken cancellationToken)
{
var counts = await db.BackgroundJobOutboxMessages.AsNoTracking()
.Where(x => x.State != BackgroundJobOutboxState.Completed)
.GroupBy(x => x.State)
.Select(group => new { State = group.Key, Count = group.Count() })
.ToListAsync(cancellationToken);
var countByState = counts.ToDictionary(x => x.State, x => x.Count);
var now = DateTime.UtcNow;
var expiredLeases = await db.BackgroundJobOutboxMessages.AsNoTracking()
.CountAsync(
x => (x.State == BackgroundJobOutboxState.Publishing ||
x.State == BackgroundJobOutboxState.Processing) &&
x.LeaseExpiresAt != null &&
x.LeaseExpiresAt < now,
cancellationToken);
var oldestUnfinishedAt = await db.BackgroundJobOutboxMessages
.AsNoTracking()
.Where(x => x.State != BackgroundJobOutboxState.Completed)
.Select(x => (DateTime?)x.CreatedAt)
.MinAsync(cancellationToken);
double? ageSeconds = oldestUnfinishedAt.HasValue
? Math.Max(0, (now - oldestUnfinishedAt.Value).TotalSeconds)
: null;
return new BackgroundJobBacklogSnapshot(
GetCount(BackgroundJobOutboxState.Pending),
GetCount(BackgroundJobOutboxState.Publishing),
GetCount(BackgroundJobOutboxState.Published),
GetCount(BackgroundJobOutboxState.Processing),
expiredLeases,
oldestUnfinishedAt,
ageSeconds);
int GetCount(BackgroundJobOutboxState state) =>
countByState.GetValueOrDefault(state);
}
}
@@ -1,3 +1,5 @@
using Jiaowu.Api.Domain.System;
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
public sealed class BackgroundJobOptions
@@ -8,13 +10,27 @@ public sealed class BackgroundJobOptions
public int PollIntervalMilliseconds { get; set; } = 500;
public int LeaseSeconds { get; set; } = 120;
public ushort PrefetchCount { get; set; } = 1;
public int AutomaticScheduleConcurrency { get; set; } = 1;
public int SchedulePublishConcurrency { get; set; } = 1;
public int MakeupExamAutoConcurrency { get; set; } = 1;
public string Exchange { get; set; } = "jiaowu.background-jobs";
public string QueuePrefix { get; set; } = "jiaowu.background-jobs";
public bool UseQuorumQueues { get; set; } = true;
public int ProcessingAttemptLimit { get; set; } = 5;
public int MaintenanceIntervalSeconds { get; set; } = 60;
public int CompletedRetentionDays { get; set; } = 14;
public int CleanupBatchSize { get; set; } = 500;
public bool UsesRabbitMq =>
Transport.Equals("RabbitMq", StringComparison.OrdinalIgnoreCase);
public int ConsumerConcurrency(BackgroundJobKind kind) => kind switch
{
BackgroundJobKind.AutomaticSchedule => AutomaticScheduleConcurrency,
BackgroundJobKind.SchedulePublish => SchedulePublishConcurrency,
BackgroundJobKind.MakeupExamAuto => MakeupExamAutoConcurrency,
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null)
};
}
public sealed class RabbitMqOptions
@@ -1,3 +1,4 @@
using System.Diagnostics;
using Jiaowu.Api.Domain.Academic;
using Jiaowu.Api.Domain.System;
using Jiaowu.Api.Infrastructure.Persistence;
@@ -9,17 +10,26 @@ public sealed class BackgroundJobOutboxPublisher(
IServiceScopeFactory scopeFactory,
IBackgroundJobTransport transport,
BackgroundJobOptions options,
BackgroundJobTelemetry telemetry,
ILogger<BackgroundJobOutboxPublisher> logger) : BackgroundService
{
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await PrepareForStartupAsync(stoppingToken);
var nextMaintenanceAt = DateTime.MinValue;
while (!stoppingToken.IsCancellationRequested)
{
try
{
await RecoverExpiredProcessingAsync(stoppingToken);
if (DateTime.UtcNow >= nextMaintenanceAt)
{
await RecoverExpiredProcessingAsync(stoppingToken);
await CleanupCompletedAsync(stoppingToken);
nextMaintenanceAt = DateTime.UtcNow.AddSeconds(
options.MaintenanceIntervalSeconds);
}
var claimed = await ClaimNextAsync(stoppingToken);
if (claimed is null)
{
@@ -62,51 +72,107 @@ public sealed class BackgroundJobOutboxPublisher(
await AddMissingOutboxMessagesAsync(db, cancellationToken);
}
private static async Task AddMissingOutboxMessagesAsync(
private async Task AddMissingOutboxMessagesAsync(
AppDbContext db,
CancellationToken cancellationToken)
{
var existing = await db.BackgroundJobOutboxMessages.AsNoTracking()
.Select(x => new { x.JobKind, x.JobId })
.ToListAsync(cancellationToken);
var existingKeys = existing
.Select(x => (x.JobKind, x.JobId))
.ToHashSet();
var automaticJobs = await db.AutomaticScheduleJobs.AsNoTracking()
.Where(x => x.Status == AutomaticScheduleJobStatus.Queued ||
x.Status == AutomaticScheduleJobStatus.Running)
.Where(x =>
(x.Status == AutomaticScheduleJobStatus.Queued ||
x.Status == AutomaticScheduleJobStatus.Running) &&
!db.BackgroundJobOutboxMessages.Any(message =>
message.JobKind == BackgroundJobKind.AutomaticSchedule &&
message.JobId == x.Id))
.Select(x => x.Id)
.ToListAsync(cancellationToken);
var publishJobs = await db.SchedulePublishJobs.AsNoTracking()
.Where(x => x.Status == SchedulePublishJobStatus.Queued ||
x.Status == SchedulePublishJobStatus.Running)
.Where(x =>
(x.Status == SchedulePublishJobStatus.Queued ||
x.Status == SchedulePublishJobStatus.Running) &&
!db.BackgroundJobOutboxMessages.Any(message =>
message.JobKind == BackgroundJobKind.SchedulePublish &&
message.JobId == x.Id))
.Select(x => x.Id)
.ToListAsync(cancellationToken);
var makeupJobs = await db.MakeupExamAutoJobs.AsNoTracking()
.Where(x => x.Status == MakeupExamAutoJobStatus.Queued ||
x.Status == MakeupExamAutoJobStatus.Running)
.Where(x =>
(x.Status == MakeupExamAutoJobStatus.Queued ||
x.Status == MakeupExamAutoJobStatus.Running) &&
!db.BackgroundJobOutboxMessages.Any(message =>
message.JobKind == BackgroundJobKind.MakeupExamAuto &&
message.JobId == x.Id))
.Select(x => x.Id)
.ToListAsync(cancellationToken);
AddMissing(BackgroundJobKind.AutomaticSchedule, automaticJobs);
AddMissing(BackgroundJobKind.SchedulePublish, publishJobs);
AddMissing(BackgroundJobKind.MakeupExamAuto, makeupJobs);
if (db.ChangeTracker.HasChanges())
await db.SaveChangesAsync(cancellationToken);
void AddMissing(BackgroundJobKind kind, IEnumerable<Guid> jobIds)
var missingKeys = automaticJobs
.Select(id => (BackgroundJobKind.AutomaticSchedule, id))
.Concat(publishJobs.Select(id =>
(BackgroundJobKind.SchedulePublish, id)))
.Concat(makeupJobs.Select(id =>
(BackgroundJobKind.MakeupExamAuto, id)))
.ToList();
foreach (var (kind, jobId) in missingKeys)
{
foreach (var jobId in jobIds)
{
if (existingKeys.Add((kind, jobId)))
{
db.BackgroundJobOutboxMessages.Add(
BackgroundJobOutboxMessage.Create(kind, jobId));
}
}
db.BackgroundJobOutboxMessages.Add(
BackgroundJobOutboxMessage.Create(kind, jobId));
}
if (!db.ChangeTracker.HasChanges())
return;
try
{
await db.SaveChangesAsync(cancellationToken);
}
catch (DbUpdateException)
{
db.ChangeTracker.Clear();
foreach (var (kind, jobId) in missingKeys)
{
var recovered = await db.BackgroundJobOutboxMessages
.AsNoTracking()
.AnyAsync(
message => message.JobKind == kind &&
message.JobId == jobId,
cancellationToken);
if (!recovered)
throw;
}
logger.LogInformation(
"Another application instance recovered {Count} missing outbox messages first.",
missingKeys.Count);
}
}
private async Task CleanupCompletedAsync(CancellationToken cancellationToken)
{
await using var scope = scopeFactory.CreateAsyncScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var cutoff = DateTime.UtcNow.AddDays(-options.CompletedRetentionDays);
var completedIds = await db.BackgroundJobOutboxMessages.AsNoTracking()
.Where(x =>
x.State == BackgroundJobOutboxState.Completed &&
x.CompletedAt != null &&
x.CompletedAt < cutoff)
.OrderBy(x => x.CompletedAt)
.Select(x => x.Id)
.Take(options.CleanupBatchSize)
.ToListAsync(cancellationToken);
if (completedIds.Count == 0)
return;
foreach (var id in completedIds)
{
db.BackgroundJobOutboxMessages.Remove(
new BackgroundJobOutboxMessage { Id = id });
}
await db.SaveChangesAsync(cancellationToken);
telemetry.RecordCleaned(completedIds.Count);
logger.LogInformation(
"Removed {Count} completed background job outbox messages older than {Cutoff}.",
completedIds.Count,
cutoff);
}
private async Task RecoverExpiredProcessingAsync(
@@ -175,6 +241,7 @@ public sealed class BackgroundJobOutboxPublisher(
Guid token,
CancellationToken cancellationToken)
{
var startedAt = Stopwatch.GetTimestamp();
try
{
await transport.PublishAsync(message, cancellationToken);
@@ -193,6 +260,11 @@ public sealed class BackgroundJobOutboxPublisher(
.SetProperty(x => x.ProcessingToken, (Guid?)null)
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null),
cancellationToken);
telemetry.RecordPublish(
message.JobKind,
transport.IsDurable,
"published",
Stopwatch.GetElapsedTime(startedAt));
}
catch (Exception exception)
{
@@ -213,6 +285,11 @@ public sealed class BackgroundJobOutboxPublisher(
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null)
.SetProperty(x => x.LastError, messageText),
cancellationToken);
telemetry.RecordPublish(
message.JobKind,
transport.IsDurable,
"failed",
Stopwatch.GetElapsedTime(startedAt));
throw;
}
}
Loaded 3 of 21 files, more files were not shown because too many files have changed in this diff. Show more