排课计算、课表发布、补考自动编排三类任务统一接入消息系统。
业务任务与 Outbox 消息同一事务落库，避免“数据库成功但消息没发出去”。 RabbitMQ 使用持久化消息、发布确认、手动 ACK、Quorum Queue、死信队列。 增加处理租约、心跳、异常重试、最大重试次数和幂等状态控制。 保留 InMemory 模式，开发环境无需安装 RabbitMQ。 支持多实例竞争消费，后续可以横向扩容。 新增 /health/messaging 消息系统健康检查。
This commit is contained in:
@@ -0,0 +1,219 @@
|
||||
using Jiaowu.Api.Domain.Academic;
|
||||
using Jiaowu.Api.Domain.System;
|
||||
using Jiaowu.Api.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||
|
||||
public sealed class BackgroundJobOutboxPublisher(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IBackgroundJobTransport transport,
|
||||
BackgroundJobOptions options,
|
||||
ILogger<BackgroundJobOutboxPublisher> logger) : BackgroundService
|
||||
{
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
await PrepareForStartupAsync(stoppingToken);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await RecoverExpiredProcessingAsync(stoppingToken);
|
||||
var claimed = await ClaimNextAsync(stoppingToken);
|
||||
if (claimed is null)
|
||||
{
|
||||
await Task.Delay(options.PollIntervalMilliseconds, stoppingToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
await PublishClaimedAsync(claimed.Value.Message, claimed.Value.Token,
|
||||
stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(exception, "Background job outbox publishing failed.");
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task PrepareForStartupAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
|
||||
if (!transport.IsDurable)
|
||||
{
|
||||
await db.BackgroundJobOutboxMessages
|
||||
.Where(x => x.State != BackgroundJobOutboxState.Completed)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(x => x.State, BackgroundJobOutboxState.Pending)
|
||||
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null),
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
await AddMissingOutboxMessagesAsync(db, cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task AddMissingOutboxMessagesAsync(
|
||||
AppDbContext db,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var existing = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||
.Select(x => new { x.JobKind, x.JobId })
|
||||
.ToListAsync(cancellationToken);
|
||||
var existingKeys = existing
|
||||
.Select(x => (x.JobKind, x.JobId))
|
||||
.ToHashSet();
|
||||
|
||||
var automaticJobs = await db.AutomaticScheduleJobs.AsNoTracking()
|
||||
.Where(x => x.Status == AutomaticScheduleJobStatus.Queued ||
|
||||
x.Status == AutomaticScheduleJobStatus.Running)
|
||||
.Select(x => x.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
var publishJobs = await db.SchedulePublishJobs.AsNoTracking()
|
||||
.Where(x => x.Status == SchedulePublishJobStatus.Queued ||
|
||||
x.Status == SchedulePublishJobStatus.Running)
|
||||
.Select(x => x.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
var makeupJobs = await db.MakeupExamAutoJobs.AsNoTracking()
|
||||
.Where(x => x.Status == MakeupExamAutoJobStatus.Queued ||
|
||||
x.Status == MakeupExamAutoJobStatus.Running)
|
||||
.Select(x => x.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
AddMissing(BackgroundJobKind.AutomaticSchedule, automaticJobs);
|
||||
AddMissing(BackgroundJobKind.SchedulePublish, publishJobs);
|
||||
AddMissing(BackgroundJobKind.MakeupExamAuto, makeupJobs);
|
||||
|
||||
if (db.ChangeTracker.HasChanges())
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
|
||||
void AddMissing(BackgroundJobKind kind, IEnumerable<Guid> jobIds)
|
||||
{
|
||||
foreach (var jobId in jobIds)
|
||||
{
|
||||
if (existingKeys.Add((kind, jobId)))
|
||||
{
|
||||
db.BackgroundJobOutboxMessages.Add(
|
||||
BackgroundJobOutboxMessage.Create(kind, jobId));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RecoverExpiredProcessingAsync(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
var now = DateTime.UtcNow;
|
||||
var recoveredState = transport.IsDurable
|
||||
? BackgroundJobOutboxState.Published
|
||||
: BackgroundJobOutboxState.Pending;
|
||||
|
||||
await db.BackgroundJobOutboxMessages
|
||||
.Where(x =>
|
||||
(x.State == BackgroundJobOutboxState.Publishing ||
|
||||
x.State == BackgroundJobOutboxState.Processing) &&
|
||||
x.LeaseExpiresAt != null &&
|
||||
x.LeaseExpiresAt < now)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(x => x.State, recoveredState)
|
||||
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null),
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<(BackgroundJobEnvelope Message, Guid Token)?> ClaimNextAsync(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
var candidateId = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||
.Where(x => x.State == BackgroundJobOutboxState.Pending)
|
||||
.OrderBy(x => x.CreatedAt)
|
||||
.Select(x => (Guid?)x.Id)
|
||||
.FirstOrDefaultAsync(cancellationToken);
|
||||
if (!candidateId.HasValue)
|
||||
return null;
|
||||
|
||||
var token = Guid.NewGuid();
|
||||
var leaseExpiresAt = DateTime.UtcNow.AddSeconds(options.LeaseSeconds);
|
||||
var claimed = await db.BackgroundJobOutboxMessages
|
||||
.Where(x =>
|
||||
x.Id == candidateId.Value &&
|
||||
x.State == BackgroundJobOutboxState.Pending)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(x => x.State, BackgroundJobOutboxState.Publishing)
|
||||
.SetProperty(x => x.ProcessingToken, token)
|
||||
.SetProperty(x => x.LeaseExpiresAt, leaseExpiresAt)
|
||||
.SetProperty(x => x.PublishAttempts, x => x.PublishAttempts + 1)
|
||||
.SetProperty(x => x.LastError, (string?)null),
|
||||
cancellationToken);
|
||||
if (claimed == 0)
|
||||
return null;
|
||||
|
||||
var message = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||
.Where(x => x.Id == candidateId.Value)
|
||||
.Select(x => new BackgroundJobEnvelope(x.Id, x.JobKind, x.JobId))
|
||||
.SingleAsync(cancellationToken);
|
||||
return (message, token);
|
||||
}
|
||||
|
||||
private async Task PublishClaimedAsync(
|
||||
BackgroundJobEnvelope message,
|
||||
Guid token,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await transport.PublishAsync(message, cancellationToken);
|
||||
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
await db.BackgroundJobOutboxMessages
|
||||
.Where(x =>
|
||||
x.Id == message.OutboxMessageId &&
|
||||
x.State == BackgroundJobOutboxState.Publishing &&
|
||||
x.ProcessingToken == token)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(x => x.State, BackgroundJobOutboxState.Published)
|
||||
.SetProperty(x => x.PublishedAt, DateTime.UtcNow)
|
||||
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null),
|
||||
cancellationToken);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
var messageText = exception.GetBaseException().Message;
|
||||
if (messageText.Length > 2000)
|
||||
messageText = messageText[..2000];
|
||||
await db.BackgroundJobOutboxMessages
|
||||
.Where(x =>
|
||||
x.Id == message.OutboxMessageId &&
|
||||
x.State == BackgroundJobOutboxState.Publishing &&
|
||||
x.ProcessingToken == token)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(x => x.State, BackgroundJobOutboxState.Pending)
|
||||
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null)
|
||||
.SetProperty(x => x.LastError, messageText),
|
||||
cancellationToken);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user