排课计算、课表发布、补考自动编排三类任务统一接入消息系统。
业务任务与 Outbox 消息同一事务落库，避免“数据库成功但消息没发出去”。 RabbitMQ 使用持久化消息、发布确认、手动 ACK、Quorum Queue、死信队列。 增加处理租约、心跳、异常重试、最大重试次数和幂等状态控制。 保留 InMemory 模式，开发环境无需安装 RabbitMQ。 支持多实例竞争消费，后续可以横向扩容。 新增 /health/messaging 消息系统健康检查。
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||
|
||||
public sealed class BackgroundJobOptions
|
||||
{
|
||||
public const string SectionName = "BackgroundJobs";
|
||||
|
||||
public string Transport { get; set; } = "InMemory";
|
||||
public int PollIntervalMilliseconds { get; set; } = 500;
|
||||
public int LeaseSeconds { get; set; } = 120;
|
||||
public ushort PrefetchCount { get; set; } = 1;
|
||||
public string Exchange { get; set; } = "jiaowu.background-jobs";
|
||||
public string QueuePrefix { get; set; } = "jiaowu.background-jobs";
|
||||
public bool UseQuorumQueues { get; set; } = true;
|
||||
public int ProcessingAttemptLimit { get; set; } = 5;
|
||||
|
||||
public bool UsesRabbitMq =>
|
||||
Transport.Equals("RabbitMq", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public sealed class RabbitMqOptions
|
||||
{
|
||||
public const string SectionName = "RabbitMq";
|
||||
|
||||
public string HostName { get; set; } = "localhost";
|
||||
public int Port { get; set; } = 5672;
|
||||
public string UserName { get; set; } = "guest";
|
||||
public string Password { get; set; } = "guest";
|
||||
public string VirtualHost { get; set; } = "/";
|
||||
public bool UseTls { get; set; }
|
||||
public string? TlsServerName { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
using Jiaowu.Api.Domain.Academic;
|
||||
using Jiaowu.Api.Domain.System;
|
||||
using Jiaowu.Api.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||
|
||||
public sealed class BackgroundJobOutboxPublisher(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IBackgroundJobTransport transport,
|
||||
BackgroundJobOptions options,
|
||||
ILogger<BackgroundJobOutboxPublisher> logger) : BackgroundService
|
||||
{
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
await PrepareForStartupAsync(stoppingToken);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await RecoverExpiredProcessingAsync(stoppingToken);
|
||||
var claimed = await ClaimNextAsync(stoppingToken);
|
||||
if (claimed is null)
|
||||
{
|
||||
await Task.Delay(options.PollIntervalMilliseconds, stoppingToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
await PublishClaimedAsync(claimed.Value.Message, claimed.Value.Token,
|
||||
stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(exception, "Background job outbox publishing failed.");
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task PrepareForStartupAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
|
||||
if (!transport.IsDurable)
|
||||
{
|
||||
await db.BackgroundJobOutboxMessages
|
||||
.Where(x => x.State != BackgroundJobOutboxState.Completed)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(x => x.State, BackgroundJobOutboxState.Pending)
|
||||
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null),
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
await AddMissingOutboxMessagesAsync(db, cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task AddMissingOutboxMessagesAsync(
|
||||
AppDbContext db,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var existing = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||
.Select(x => new { x.JobKind, x.JobId })
|
||||
.ToListAsync(cancellationToken);
|
||||
var existingKeys = existing
|
||||
.Select(x => (x.JobKind, x.JobId))
|
||||
.ToHashSet();
|
||||
|
||||
var automaticJobs = await db.AutomaticScheduleJobs.AsNoTracking()
|
||||
.Where(x => x.Status == AutomaticScheduleJobStatus.Queued ||
|
||||
x.Status == AutomaticScheduleJobStatus.Running)
|
||||
.Select(x => x.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
var publishJobs = await db.SchedulePublishJobs.AsNoTracking()
|
||||
.Where(x => x.Status == SchedulePublishJobStatus.Queued ||
|
||||
x.Status == SchedulePublishJobStatus.Running)
|
||||
.Select(x => x.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
var makeupJobs = await db.MakeupExamAutoJobs.AsNoTracking()
|
||||
.Where(x => x.Status == MakeupExamAutoJobStatus.Queued ||
|
||||
x.Status == MakeupExamAutoJobStatus.Running)
|
||||
.Select(x => x.Id)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
AddMissing(BackgroundJobKind.AutomaticSchedule, automaticJobs);
|
||||
AddMissing(BackgroundJobKind.SchedulePublish, publishJobs);
|
||||
AddMissing(BackgroundJobKind.MakeupExamAuto, makeupJobs);
|
||||
|
||||
if (db.ChangeTracker.HasChanges())
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
|
||||
void AddMissing(BackgroundJobKind kind, IEnumerable<Guid> jobIds)
|
||||
{
|
||||
foreach (var jobId in jobIds)
|
||||
{
|
||||
if (existingKeys.Add((kind, jobId)))
|
||||
{
|
||||
db.BackgroundJobOutboxMessages.Add(
|
||||
BackgroundJobOutboxMessage.Create(kind, jobId));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RecoverExpiredProcessingAsync(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
var now = DateTime.UtcNow;
|
||||
var recoveredState = transport.IsDurable
|
||||
? BackgroundJobOutboxState.Published
|
||||
: BackgroundJobOutboxState.Pending;
|
||||
|
||||
await db.BackgroundJobOutboxMessages
|
||||
.Where(x =>
|
||||
(x.State == BackgroundJobOutboxState.Publishing ||
|
||||
x.State == BackgroundJobOutboxState.Processing) &&
|
||||
x.LeaseExpiresAt != null &&
|
||||
x.LeaseExpiresAt < now)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(x => x.State, recoveredState)
|
||||
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null),
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<(BackgroundJobEnvelope Message, Guid Token)?> ClaimNextAsync(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
var candidateId = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||
.Where(x => x.State == BackgroundJobOutboxState.Pending)
|
||||
.OrderBy(x => x.CreatedAt)
|
||||
.Select(x => (Guid?)x.Id)
|
||||
.FirstOrDefaultAsync(cancellationToken);
|
||||
if (!candidateId.HasValue)
|
||||
return null;
|
||||
|
||||
var token = Guid.NewGuid();
|
||||
var leaseExpiresAt = DateTime.UtcNow.AddSeconds(options.LeaseSeconds);
|
||||
var claimed = await db.BackgroundJobOutboxMessages
|
||||
.Where(x =>
|
||||
x.Id == candidateId.Value &&
|
||||
x.State == BackgroundJobOutboxState.Pending)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(x => x.State, BackgroundJobOutboxState.Publishing)
|
||||
.SetProperty(x => x.ProcessingToken, token)
|
||||
.SetProperty(x => x.LeaseExpiresAt, leaseExpiresAt)
|
||||
.SetProperty(x => x.PublishAttempts, x => x.PublishAttempts + 1)
|
||||
.SetProperty(x => x.LastError, (string?)null),
|
||||
cancellationToken);
|
||||
if (claimed == 0)
|
||||
return null;
|
||||
|
||||
var message = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||
.Where(x => x.Id == candidateId.Value)
|
||||
.Select(x => new BackgroundJobEnvelope(x.Id, x.JobKind, x.JobId))
|
||||
.SingleAsync(cancellationToken);
|
||||
return (message, token);
|
||||
}
|
||||
|
||||
private async Task PublishClaimedAsync(
|
||||
BackgroundJobEnvelope message,
|
||||
Guid token,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await transport.PublishAsync(message, cancellationToken);
|
||||
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
await db.BackgroundJobOutboxMessages
|
||||
.Where(x =>
|
||||
x.Id == message.OutboxMessageId &&
|
||||
x.State == BackgroundJobOutboxState.Publishing &&
|
||||
x.ProcessingToken == token)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(x => x.State, BackgroundJobOutboxState.Published)
|
||||
.SetProperty(x => x.PublishedAt, DateTime.UtcNow)
|
||||
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null),
|
||||
cancellationToken);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
var messageText = exception.GetBaseException().Message;
|
||||
if (messageText.Length > 2000)
|
||||
messageText = messageText[..2000];
|
||||
await db.BackgroundJobOutboxMessages
|
||||
.Where(x =>
|
||||
x.Id == message.OutboxMessageId &&
|
||||
x.State == BackgroundJobOutboxState.Publishing &&
|
||||
x.ProcessingToken == token)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(x => x.State, BackgroundJobOutboxState.Pending)
|
||||
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null)
|
||||
.SetProperty(x => x.LastError, messageText),
|
||||
cancellationToken);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
using Jiaowu.Api.Domain.Academic;
|
||||
using Jiaowu.Api.Domain.System;
|
||||
using Jiaowu.Api.Infrastructure.Exams;
|
||||
using Jiaowu.Api.Infrastructure.Persistence;
|
||||
using Jiaowu.Api.Infrastructure.Scheduling;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||
|
||||
public sealed record BackgroundJobRunResult(
|
||||
BackgroundJobRunOutcome Outcome,
|
||||
TimeSpan RetryAfter)
|
||||
{
|
||||
public static BackgroundJobRunResult Completed { get; } =
|
||||
new(BackgroundJobRunOutcome.Completed, TimeSpan.Zero);
|
||||
|
||||
public static BackgroundJobRunResult Retry(TimeSpan retryAfter) =>
|
||||
new(BackgroundJobRunOutcome.Retry, retryAfter);
|
||||
}
|
||||
|
||||
public enum BackgroundJobRunOutcome
|
||||
{
|
||||
Completed,
|
||||
Retry
|
||||
}
|
||||
|
||||
public sealed class BackgroundJobRunner(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IBackgroundJobTransport transport,
|
||||
BackgroundJobOptions options,
|
||||
ILogger<BackgroundJobRunner> logger)
|
||||
{
|
||||
public async Task<BackgroundJobRunResult> RunAsync(
|
||||
BackgroundJobEnvelope message,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var token = Guid.NewGuid();
|
||||
var claim = await TryClaimAsync(message, token, cancellationToken);
|
||||
if (!claim.Claimed)
|
||||
{
|
||||
return claim.Completed
|
||||
? BackgroundJobRunResult.Completed
|
||||
: BackgroundJobRunResult.Retry(claim.RetryAfter);
|
||||
}
|
||||
|
||||
if (claim.AttemptsExceeded)
|
||||
{
|
||||
await MarkJobRetryLimitExceededAsync(message, cancellationToken);
|
||||
await MarkCompletedAsync(message.OutboxMessageId, token,
|
||||
cancellationToken);
|
||||
return BackgroundJobRunResult.Completed;
|
||||
}
|
||||
|
||||
using var heartbeatCancellation =
|
||||
CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
var heartbeat = RunHeartbeatAsync(
|
||||
message.OutboxMessageId,
|
||||
token,
|
||||
heartbeatCancellation.Token);
|
||||
|
||||
try
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
switch (message.JobKind)
|
||||
{
|
||||
case BackgroundJobKind.AutomaticSchedule:
|
||||
await scope.ServiceProvider
|
||||
.GetRequiredService<AutomaticScheduleJobProcessor>()
|
||||
.ProcessAsync(message.JobId, cancellationToken);
|
||||
break;
|
||||
case BackgroundJobKind.SchedulePublish:
|
||||
await scope.ServiceProvider
|
||||
.GetRequiredService<SchedulePublishJobProcessor>()
|
||||
.ProcessAsync(message.JobId, cancellationToken);
|
||||
break;
|
||||
case BackgroundJobKind.MakeupExamAuto:
|
||||
await scope.ServiceProvider
|
||||
.GetRequiredService<MakeupExamAutoJobProcessor>()
|
||||
.ProcessAsync(message.JobId, cancellationToken);
|
||||
break;
|
||||
default:
|
||||
throw new InvalidOperationException(
|
||||
$"Unsupported background job kind '{message.JobKind}'.");
|
||||
}
|
||||
|
||||
await MarkCompletedAsync(message.OutboxMessageId, token,
|
||||
cancellationToken);
|
||||
return BackgroundJobRunResult.Completed;
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Background job {JobKind}/{JobId} will be retried.",
|
||||
message.JobKind,
|
||||
message.JobId);
|
||||
await ReleaseForRetryAsync(
|
||||
message.OutboxMessageId,
|
||||
token,
|
||||
exception,
|
||||
CancellationToken.None);
|
||||
return BackgroundJobRunResult.Retry(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
finally
|
||||
{
|
||||
await heartbeatCancellation.CancelAsync();
|
||||
try
|
||||
{
|
||||
await heartbeat;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Expected when processing completes or the application stops.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<(
|
||||
bool Claimed,
|
||||
bool Completed,
|
||||
bool AttemptsExceeded,
|
||||
TimeSpan RetryAfter)>
|
||||
TryClaimAsync(
|
||||
BackgroundJobEnvelope message,
|
||||
Guid token,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
var now = DateTime.UtcNow;
|
||||
var leaseExpiresAt = now.AddSeconds(options.LeaseSeconds);
|
||||
var claimed = await db.BackgroundJobOutboxMessages
|
||||
.Where(x =>
|
||||
x.Id == message.OutboxMessageId &&
|
||||
x.JobKind == message.JobKind &&
|
||||
x.JobId == message.JobId &&
|
||||
(x.State == BackgroundJobOutboxState.Pending ||
|
||||
x.State == BackgroundJobOutboxState.Publishing ||
|
||||
x.State == BackgroundJobOutboxState.Published ||
|
||||
(x.State == BackgroundJobOutboxState.Processing &&
|
||||
x.LeaseExpiresAt != null &&
|
||||
x.LeaseExpiresAt < now)))
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(x => x.State, BackgroundJobOutboxState.Processing)
|
||||
.SetProperty(x => x.ProcessingToken, token)
|
||||
.SetProperty(x => x.LeaseExpiresAt, leaseExpiresAt)
|
||||
.SetProperty(
|
||||
x => x.ProcessingAttempts,
|
||||
x => x.ProcessingAttempts + 1)
|
||||
.SetProperty(x => x.LastError, (string?)null),
|
||||
cancellationToken);
|
||||
if (claimed == 1)
|
||||
{
|
||||
var attempts = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||
.Where(x => x.Id == message.OutboxMessageId)
|
||||
.Select(x => x.ProcessingAttempts)
|
||||
.SingleAsync(cancellationToken);
|
||||
return (
|
||||
true,
|
||||
false,
|
||||
attempts > options.ProcessingAttemptLimit,
|
||||
TimeSpan.Zero);
|
||||
}
|
||||
|
||||
var current = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||
.Where(x => x.Id == message.OutboxMessageId)
|
||||
.Select(x => new { x.State, x.LeaseExpiresAt })
|
||||
.FirstOrDefaultAsync(cancellationToken);
|
||||
if (current is null || current.State == BackgroundJobOutboxState.Completed)
|
||||
return (false, true, false, TimeSpan.Zero);
|
||||
|
||||
var retryAfter = current.LeaseExpiresAt.HasValue
|
||||
? current.LeaseExpiresAt.Value - now
|
||||
: TimeSpan.FromSeconds(2);
|
||||
retryAfter = TimeSpan.FromSeconds(Math.Clamp(
|
||||
retryAfter.TotalSeconds,
|
||||
1,
|
||||
options.LeaseSeconds));
|
||||
return (false, false, false, retryAfter);
|
||||
}
|
||||
|
||||
private async Task MarkJobRetryLimitExceededAsync(
|
||||
BackgroundJobEnvelope message,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
var completedAt = DateTime.UtcNow;
|
||||
var error = $"后台任务连续处理失败超过 {options.ProcessingAttemptLimit} 次，" +
|
||||
"已停止自动重试，请检查服务日志后重新创建任务。";
|
||||
|
||||
switch (message.JobKind)
|
||||
{
|
||||
case BackgroundJobKind.AutomaticSchedule:
|
||||
await db.AutomaticScheduleJobs
|
||||
.Where(x =>
|
||||
x.Id == message.JobId &&
|
||||
x.Status != AutomaticScheduleJobStatus.Succeeded &&
|
||||
x.Status != AutomaticScheduleJobStatus.Failed)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(
|
||||
x => x.Status,
|
||||
AutomaticScheduleJobStatus.Failed)
|
||||
.SetProperty(x => x.ActiveSchedulePlanId, (Guid?)null)
|
||||
.SetProperty(x => x.ErrorMessage, error)
|
||||
.SetProperty(x => x.CompletedAt, completedAt),
|
||||
cancellationToken);
|
||||
break;
|
||||
case BackgroundJobKind.SchedulePublish:
|
||||
await db.SchedulePublishJobs
|
||||
.Where(x =>
|
||||
x.Id == message.JobId &&
|
||||
x.Status != SchedulePublishJobStatus.Succeeded &&
|
||||
x.Status != SchedulePublishJobStatus.Failed)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(
|
||||
x => x.Status,
|
||||
SchedulePublishJobStatus.Failed)
|
||||
.SetProperty(x => x.ActiveAcademicTermId, (Guid?)null)
|
||||
.SetProperty(x => x.CurrentStep, "后台处理已停止")
|
||||
.SetProperty(x => x.ErrorMessage, error)
|
||||
.SetProperty(x => x.CompletedAt, completedAt),
|
||||
cancellationToken);
|
||||
break;
|
||||
case BackgroundJobKind.MakeupExamAuto:
|
||||
await db.MakeupExamAutoJobs
|
||||
.Where(x =>
|
||||
x.Id == message.JobId &&
|
||||
x.Status != MakeupExamAutoJobStatus.Succeeded &&
|
||||
x.Status != MakeupExamAutoJobStatus.Failed)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(
|
||||
x => x.Status,
|
||||
MakeupExamAutoJobStatus.Failed)
|
||||
.SetProperty(x => x.ErrorMessage, error)
|
||||
.SetProperty(x => x.CompletedAt, completedAt),
|
||||
cancellationToken);
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException(
|
||||
nameof(message.JobKind),
|
||||
message.JobKind,
|
||||
null);
|
||||
}
|
||||
|
||||
logger.LogError(
|
||||
"Background job {JobKind}/{JobId} exceeded {AttemptLimit} processing attempts.",
|
||||
message.JobKind,
|
||||
message.JobId,
|
||||
options.ProcessingAttemptLimit);
|
||||
}
|
||||
|
||||
private async Task RunHeartbeatAsync(
|
||||
Guid outboxMessageId,
|
||||
Guid token,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var intervalSeconds = Math.Max(5, options.LeaseSeconds / 3);
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(intervalSeconds), cancellationToken);
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
await db.BackgroundJobOutboxMessages
|
||||
.Where(x =>
|
||||
x.Id == outboxMessageId &&
|
||||
x.State == BackgroundJobOutboxState.Processing &&
|
||||
x.ProcessingToken == token)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters.SetProperty(
|
||||
x => x.LeaseExpiresAt,
|
||||
DateTime.UtcNow.AddSeconds(options.LeaseSeconds)),
|
||||
cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task MarkCompletedAsync(
|
||||
Guid outboxMessageId,
|
||||
Guid token,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
await db.BackgroundJobOutboxMessages
|
||||
.Where(x =>
|
||||
x.Id == outboxMessageId &&
|
||||
x.State == BackgroundJobOutboxState.Processing &&
|
||||
x.ProcessingToken == token)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(x => x.State, BackgroundJobOutboxState.Completed)
|
||||
.SetProperty(x => x.CompletedAt, DateTime.UtcNow)
|
||||
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null),
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private async Task ReleaseForRetryAsync(
|
||||
Guid outboxMessageId,
|
||||
Guid token,
|
||||
Exception exception,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
var message = exception.GetBaseException().Message;
|
||||
if (message.Length > 2000)
|
||||
message = message[..2000];
|
||||
var state = transport.IsDurable
|
||||
? BackgroundJobOutboxState.Published
|
||||
: BackgroundJobOutboxState.Pending;
|
||||
await db.BackgroundJobOutboxMessages
|
||||
.Where(x =>
|
||||
x.Id == outboxMessageId &&
|
||||
x.State == BackgroundJobOutboxState.Processing &&
|
||||
x.ProcessingToken == token)
|
||||
.ExecuteUpdateAsync(
|
||||
setters => setters
|
||||
.SetProperty(x => x.State, state)
|
||||
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null)
|
||||
.SetProperty(x => x.LastError, message),
|
||||
cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
using System.Threading.Channels;
|
||||
using Jiaowu.Api.Domain.System;
|
||||
|
||||
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||
|
||||
public sealed record BackgroundJobEnvelope(
|
||||
Guid OutboxMessageId,
|
||||
BackgroundJobKind JobKind,
|
||||
Guid JobId);
|
||||
|
||||
public interface IBackgroundJobTransport
|
||||
{
|
||||
bool IsDurable { get; }
|
||||
|
||||
ValueTask PublishAsync(
|
||||
BackgroundJobEnvelope message,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task<bool> CheckHealthAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class InMemoryBackgroundJobTransport : IBackgroundJobTransport
|
||||
{
|
||||
private readonly Channel<BackgroundJobEnvelope> _channel =
|
||||
Channel.CreateBounded<BackgroundJobEnvelope>(
|
||||
new BoundedChannelOptions(256)
|
||||
{
|
||||
FullMode = BoundedChannelFullMode.Wait,
|
||||
SingleReader = true,
|
||||
SingleWriter = false
|
||||
});
|
||||
|
||||
public bool IsDurable => false;
|
||||
|
||||
public ValueTask PublishAsync(
|
||||
BackgroundJobEnvelope message,
|
||||
CancellationToken cancellationToken) =>
|
||||
_channel.Writer.WriteAsync(message, cancellationToken);
|
||||
|
||||
public Task<bool> CheckHealthAsync(CancellationToken cancellationToken) =>
|
||||
Task.FromResult(true);
|
||||
|
||||
public IAsyncEnumerable<BackgroundJobEnvelope> ReadAllAsync(
|
||||
CancellationToken cancellationToken) =>
|
||||
_channel.Reader.ReadAllAsync(cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||
|
||||
public sealed class InMemoryBackgroundJobWorker(
|
||||
InMemoryBackgroundJobTransport transport,
|
||||
BackgroundJobRunner runner,
|
||||
ILogger<InMemoryBackgroundJobWorker> logger) : BackgroundService
|
||||
{
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await foreach (var message in transport.ReadAllAsync(stoppingToken))
|
||||
{
|
||||
var result = await runner.RunAsync(message, stoppingToken);
|
||||
if (result.Outcome != BackgroundJobRunOutcome.Retry)
|
||||
continue;
|
||||
|
||||
await Task.Delay(result.RetryAfter, stoppingToken);
|
||||
await transport.PublishAsync(message, stoppingToken);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
logger.LogInformation("In-memory background job worker is stopping.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,416 @@
|
||||
using System.Text.Json;
|
||||
using Jiaowu.Api.Domain.System;
|
||||
using RabbitMQ.Client;
|
||||
using RabbitMQ.Client.Events;
|
||||
|
||||
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||
|
||||
public sealed class RabbitMqBackgroundJobTransport(
|
||||
BackgroundJobOptions jobOptions,
|
||||
RabbitMqOptions rabbitOptions,
|
||||
ILogger<RabbitMqBackgroundJobTransport> logger)
|
||||
: IBackgroundJobTransport, IAsyncDisposable
|
||||
{
|
||||
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||
private IConnection? _connection;
|
||||
private IChannel? _channel;
|
||||
|
||||
public bool IsDurable => true;
|
||||
|
||||
public async ValueTask PublishAsync(
|
||||
BackgroundJobEnvelope message,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var body = JsonSerializer.SerializeToUtf8Bytes(message);
|
||||
await _gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
var channel = await GetChannelAsync(cancellationToken);
|
||||
var properties = new BasicProperties
|
||||
{
|
||||
Persistent = true,
|
||||
ContentType = "application/json",
|
||||
MessageId = message.OutboxMessageId.ToString("D"),
|
||||
Type = message.JobKind.ToString(),
|
||||
AppId = "jiaowu-api"
|
||||
};
|
||||
await channel.BasicPublishAsync(
|
||||
jobOptions.Exchange,
|
||||
RabbitMqBackgroundJobTopology.RoutingKey(message.JobKind),
|
||||
mandatory: true,
|
||||
properties,
|
||||
body,
|
||||
cancellationToken);
|
||||
}
|
||||
catch
|
||||
{
|
||||
await ResetConnectionAsync();
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<bool> CheckHealthAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await _gate.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
var channel = await GetChannelAsync(cancellationToken);
|
||||
return channel.IsOpen;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "RabbitMQ health check failed.");
|
||||
await ResetConnectionAsync();
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IChannel> GetChannelAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_channel is { IsOpen: true })
|
||||
return _channel;
|
||||
|
||||
await ResetConnectionAsync();
|
||||
_connection = await RabbitMqBackgroundJobTopology.CreateConnectionAsync(
|
||||
rabbitOptions,
|
||||
"jiaowu-background-job-publisher",
|
||||
cancellationToken);
|
||||
_channel = await _connection.CreateChannelAsync(
|
||||
new CreateChannelOptions(
|
||||
publisherConfirmationsEnabled: true,
|
||||
publisherConfirmationTrackingEnabled: true),
|
||||
cancellationToken);
|
||||
await RabbitMqBackgroundJobTopology.DeclareAsync(
|
||||
_channel,
|
||||
jobOptions,
|
||||
cancellationToken);
|
||||
return _channel;
|
||||
}
|
||||
|
||||
private async Task ResetConnectionAsync()
|
||||
{
|
||||
if (_channel is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _channel.DisposeAsync();
|
||||
}
|
||||
catch
|
||||
{
|
||||
// The broker may already have closed the channel.
|
||||
}
|
||||
_channel = null;
|
||||
}
|
||||
|
||||
if (_connection is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _connection.DisposeAsync();
|
||||
}
|
||||
catch
|
||||
{
|
||||
// The broker may already have closed the connection.
|
||||
}
|
||||
_connection = null;
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _gate.WaitAsync();
|
||||
try
|
||||
{
|
||||
await ResetConnectionAsync();
|
||||
}
|
||||
finally
|
||||
{
|
||||
_gate.Release();
|
||||
_gate.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class RabbitMqBackgroundJobWorker(
|
||||
BackgroundJobOptions jobOptions,
|
||||
RabbitMqOptions rabbitOptions,
|
||||
BackgroundJobRunner runner,
|
||||
ILogger<RabbitMqBackgroundJobWorker> logger) : BackgroundService
|
||||
{
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
IConnection? connection = null;
|
||||
var channels = new List<IChannel>();
|
||||
try
|
||||
{
|
||||
connection = await RabbitMqBackgroundJobTopology.CreateConnectionAsync(
|
||||
rabbitOptions,
|
||||
"jiaowu-background-job-worker",
|
||||
stoppingToken);
|
||||
|
||||
foreach (var kind in RabbitMqBackgroundJobTopology.JobKinds)
|
||||
{
|
||||
var channel = await connection.CreateChannelAsync(
|
||||
new CreateChannelOptions(
|
||||
publisherConfirmationsEnabled: false,
|
||||
publisherConfirmationTrackingEnabled: false),
|
||||
stoppingToken);
|
||||
channels.Add(channel);
|
||||
await RabbitMqBackgroundJobTopology.DeclareAsync(
|
||||
channel,
|
||||
jobOptions,
|
||||
stoppingToken);
|
||||
await channel.BasicQosAsync(
|
||||
0,
|
||||
jobOptions.PrefetchCount,
|
||||
global: false,
|
||||
stoppingToken);
|
||||
|
||||
var consumer = new AsyncEventingBasicConsumer(channel);
|
||||
consumer.ReceivedAsync += async (_, eventArgs) =>
|
||||
{
|
||||
using var deliveryCancellation =
|
||||
CancellationTokenSource.CreateLinkedTokenSource(
|
||||
eventArgs.CancellationToken,
|
||||
stoppingToken);
|
||||
var deliveryToken = deliveryCancellation.Token;
|
||||
try
|
||||
{
|
||||
var message = JsonSerializer.Deserialize<BackgroundJobEnvelope>(
|
||||
eventArgs.Body.Span);
|
||||
if (message is null || message.JobKind != kind)
|
||||
{
|
||||
await channel.BasicNackAsync(
|
||||
eventArgs.DeliveryTag,
|
||||
multiple: false,
|
||||
requeue: false,
|
||||
deliveryToken);
|
||||
return;
|
||||
}
|
||||
|
||||
var result = await runner.RunAsync(
|
||||
message,
|
||||
deliveryToken);
|
||||
if (result.Outcome == BackgroundJobRunOutcome.Completed)
|
||||
{
|
||||
await channel.BasicAckAsync(
|
||||
eventArgs.DeliveryTag,
|
||||
multiple: false,
|
||||
deliveryToken);
|
||||
return;
|
||||
}
|
||||
|
||||
await Task.Delay(
|
||||
result.RetryAfter,
|
||||
deliveryToken);
|
||||
await channel.BasicNackAsync(
|
||||
eventArgs.DeliveryTag,
|
||||
multiple: false,
|
||||
requeue: true,
|
||||
deliveryToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Closing the channel requeues unacknowledged deliveries.
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"RabbitMQ delivery for {JobKind} failed and will be requeued.",
|
||||
kind);
|
||||
if (channel.IsOpen)
|
||||
{
|
||||
await channel.BasicNackAsync(
|
||||
eventArgs.DeliveryTag,
|
||||
multiple: false,
|
||||
requeue: true,
|
||||
CancellationToken.None);
|
||||
}
|
||||
}
|
||||
};
|
||||
await channel.BasicConsumeAsync(
|
||||
RabbitMqBackgroundJobTopology.QueueName(jobOptions, kind),
|
||||
autoAck: false,
|
||||
consumer,
|
||||
stoppingToken);
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"RabbitMQ background job consumers are connected to {HostName}:{Port}.",
|
||||
rabbitOptions.HostName,
|
||||
rabbitOptions.Port);
|
||||
while (connection.IsOpen && !stoppingToken.IsCancellationRequested)
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"RabbitMQ background job consumer connection failed; retrying.");
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (var channel in channels)
|
||||
{
|
||||
try
|
||||
{
|
||||
await channel.DisposeAsync();
|
||||
}
|
||||
catch
|
||||
{
|
||||
// The connection may already have disposed its channels.
|
||||
}
|
||||
}
|
||||
|
||||
if (connection is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
await connection.DisposeAsync();
|
||||
}
|
||||
catch
|
||||
{
|
||||
// The broker may already have closed the connection.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!stoppingToken.IsCancellationRequested)
|
||||
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal static class RabbitMqBackgroundJobTopology
|
||||
{
|
||||
public static readonly BackgroundJobKind[] JobKinds =
|
||||
[
|
||||
BackgroundJobKind.AutomaticSchedule,
|
||||
BackgroundJobKind.SchedulePublish,
|
||||
BackgroundJobKind.MakeupExamAuto
|
||||
];
|
||||
|
||||
public static async Task<IConnection> CreateConnectionAsync(
|
||||
RabbitMqOptions options,
|
||||
string clientName,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var factory = new ConnectionFactory
|
||||
{
|
||||
HostName = options.HostName,
|
||||
Port = options.Port,
|
||||
UserName = options.UserName,
|
||||
Password = options.Password,
|
||||
VirtualHost = options.VirtualHost,
|
||||
ClientProvidedName = clientName,
|
||||
AutomaticRecoveryEnabled = false,
|
||||
TopologyRecoveryEnabled = false,
|
||||
RequestedHeartbeat = TimeSpan.FromSeconds(30),
|
||||
ConsumerDispatchConcurrency = 1
|
||||
};
|
||||
if (options.UseTls)
|
||||
{
|
||||
factory.Ssl = new SslOption
|
||||
{
|
||||
Enabled = true,
|
||||
ServerName = string.IsNullOrWhiteSpace(options.TlsServerName)
|
||||
? options.HostName
|
||||
: options.TlsServerName
|
||||
};
|
||||
}
|
||||
|
||||
return await factory.CreateConnectionAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public static async Task DeclareAsync(
|
||||
IChannel channel,
|
||||
BackgroundJobOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var deadExchange = options.Exchange + ".dead";
|
||||
await channel.ExchangeDeclareAsync(
|
||||
options.Exchange,
|
||||
ExchangeType.Direct,
|
||||
durable: true,
|
||||
autoDelete: false,
|
||||
cancellationToken: cancellationToken);
|
||||
await channel.ExchangeDeclareAsync(
|
||||
deadExchange,
|
||||
ExchangeType.Direct,
|
||||
durable: true,
|
||||
autoDelete: false,
|
||||
cancellationToken: cancellationToken);
|
||||
|
||||
foreach (var kind in JobKinds)
|
||||
{
|
||||
var routingKey = RoutingKey(kind);
|
||||
var queueName = QueueName(options, kind);
|
||||
var deadQueueName = queueName + ".dead";
|
||||
var queueArguments = QueueArguments(options);
|
||||
queueArguments["x-dead-letter-exchange"] = deadExchange;
|
||||
queueArguments["x-dead-letter-routing-key"] = routingKey;
|
||||
await channel.QueueDeclareAsync(
|
||||
queueName,
|
||||
durable: true,
|
||||
exclusive: false,
|
||||
autoDelete: false,
|
||||
arguments: queueArguments,
|
||||
cancellationToken: cancellationToken);
|
||||
await channel.QueueBindAsync(
|
||||
queueName,
|
||||
options.Exchange,
|
||||
routingKey,
|
||||
cancellationToken: cancellationToken);
|
||||
|
||||
await channel.QueueDeclareAsync(
|
||||
deadQueueName,
|
||||
durable: true,
|
||||
exclusive: false,
|
||||
autoDelete: false,
|
||||
arguments: QueueArguments(options),
|
||||
cancellationToken: cancellationToken);
|
||||
await channel.QueueBindAsync(
|
||||
deadQueueName,
|
||||
deadExchange,
|
||||
routingKey,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
public static string RoutingKey(BackgroundJobKind kind) => kind switch
|
||||
{
|
||||
BackgroundJobKind.AutomaticSchedule => "schedule.automatic",
|
||||
BackgroundJobKind.SchedulePublish => "schedule.publish",
|
||||
BackgroundJobKind.MakeupExamAuto => "makeup-exam.automatic",
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null)
|
||||
};
|
||||
|
||||
public static string QueueName(
|
||||
BackgroundJobOptions options,
|
||||
BackgroundJobKind kind) =>
|
||||
$"{options.QueuePrefix}.{RoutingKey(kind)}";
|
||||
|
||||
private static Dictionary<string, object?> QueueArguments(
|
||||
BackgroundJobOptions options)
|
||||
{
|
||||
var arguments = new Dictionary<string, object?>();
|
||||
if (options.UseQuorumQueues)
|
||||
arguments["x-queue-type"] = "quorum";
|
||||
return arguments;
|
||||
}
|
||||
}
|
||||
@@ -1,111 +1,10 @@
|
||||
using System.Text.Json;
|
||||
using System.Threading.Channels;
|
||||
using Jiaowu.Api.Domain.Academic;
|
||||
using Jiaowu.Api.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace Jiaowu.Api.Infrastructure.Exams;
|
||||
|
||||
public sealed class MakeupExamAutoJobQueue
|
||||
{
|
||||
private readonly Channel<Guid> _channel = Channel.CreateUnbounded<Guid>(
|
||||
new UnboundedChannelOptions
|
||||
{
|
||||
SingleReader = true,
|
||||
SingleWriter = false
|
||||
});
|
||||
|
||||
public void Enqueue(Guid jobId)
|
||||
{
|
||||
if (!_channel.Writer.TryWrite(jobId))
|
||||
throw new InvalidOperationException("补考自动生成任务队列当前不可用。");
|
||||
}
|
||||
|
||||
public IAsyncEnumerable<Guid> ReadAllAsync(CancellationToken cancellationToken) =>
|
||||
_channel.Reader.ReadAllAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class MakeupExamAutoJobWorker(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
MakeupExamAutoJobQueue queue,
|
||||
ILogger<MakeupExamAutoJobWorker> logger) : BackgroundService
|
||||
{
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await RecoverInterruptedJobsAsync(stoppingToken);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Could not recover makeup exam auto jobs (table may not exist yet).");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await foreach (var jobId in queue.ReadAllAsync(stoppingToken))
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var processor = scope.ServiceProvider
|
||||
.GetRequiredService<MakeupExamAutoJobProcessor>();
|
||||
await processor.ProcessAsync(jobId, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Unexpected failure while dispatching makeup exam auto job {JobId}.",
|
||||
jobId);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
logger.LogInformation("Makeup exam auto job worker is stopping.");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RecoverInterruptedJobsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
var jobs = await db.MakeupExamAutoJobs
|
||||
.Where(x =>
|
||||
x.Status == MakeupExamAutoJobStatus.Queued ||
|
||||
x.Status == MakeupExamAutoJobStatus.Running)
|
||||
.OrderBy(x => x.CreatedAt)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
foreach (var job in jobs)
|
||||
{
|
||||
job.Status = MakeupExamAutoJobStatus.Queued;
|
||||
job.StartedAt = null;
|
||||
job.CompletedAt = null;
|
||||
job.ErrorMessage = null;
|
||||
}
|
||||
|
||||
if (jobs.Count > 0)
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
foreach (var job in jobs)
|
||||
queue.Enqueue(job.Id);
|
||||
|
||||
if (jobs.Count > 0)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Recovered {JobCount} queued or interrupted makeup exam auto jobs.",
|
||||
jobs.Count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class MakeupExamAutoJobProcessor(
|
||||
AppDbContext db,
|
||||
MakeupExamEligibilityService eligibilityService,
|
||||
|
||||
@@ -98,6 +98,8 @@ public sealed class AppDbContext(DbContextOptions<AppDbContext> options)
|
||||
public DbSet<OfficialDocumentDownload> OfficialDocumentDownloads =>
|
||||
Set<OfficialDocumentDownload>();
|
||||
public DbSet<AuditLog> AuditLogs => Set<AuditLog>();
|
||||
public DbSet<BackgroundJobOutboxMessage> BackgroundJobOutboxMessages =>
|
||||
Set<BackgroundJobOutboxMessage>();
|
||||
|
||||
protected override void ConfigureConventions(
|
||||
ModelConfigurationBuilder configurationBuilder)
|
||||
@@ -993,6 +995,14 @@ public sealed class AppDbContext(DbContextOptions<AppDbContext> options)
|
||||
entity.HasIndex(x => x.CreatedAt);
|
||||
});
|
||||
|
||||
builder.Entity<BackgroundJobOutboxMessage>(entity =>
|
||||
{
|
||||
entity.Property(x => x.LastError).HasMaxLength(2000);
|
||||
entity.HasIndex(x => new { x.JobKind, x.JobId }).IsUnique();
|
||||
entity.HasIndex(x => new { x.State, x.CreatedAt });
|
||||
entity.HasIndex(x => x.LeaseExpiresAt);
|
||||
});
|
||||
|
||||
builder.Entity<OfficialDocument>(entity =>
|
||||
{
|
||||
entity.Property(x => x.DocumentNumber).HasMaxLength(50);
|
||||
|
||||
@@ -58,6 +58,8 @@ public sealed class DevelopmentSqliteMigrator(
|
||||
"20260726_31_unified_message_center";
|
||||
private const string ClassroomReservationsMigration =
|
||||
"20260726_32_classroom_reservations";
|
||||
private const string BackgroundJobOutboxMigration =
|
||||
"20260726_33_background_job_outbox";
|
||||
|
||||
public async Task MigrateAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
@@ -421,6 +423,19 @@ public sealed class DevelopmentSqliteMigrator(
|
||||
ClassroomReservationsMigration,
|
||||
classroomReservationsExist ? [] : ClassroomReservationStatements,
|
||||
cancellationToken);
|
||||
|
||||
var backgroundJobOutboxExists = await db.Database
|
||||
.SqlQueryRaw<int>(
|
||||
"""
|
||||
SELECT COUNT(*) AS "Value"
|
||||
FROM sqlite_master
|
||||
WHERE type = 'table' AND name = 'BackgroundJobOutboxMessages'
|
||||
""")
|
||||
.AnyAsync(value => value > 0, cancellationToken);
|
||||
await ApplyMigrationAsync(
|
||||
BackgroundJobOutboxMigration,
|
||||
backgroundJobOutboxExists ? [] : BackgroundJobOutboxStatements,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private async Task ApplyMigrationAsync(
|
||||
@@ -2004,4 +2019,37 @@ public sealed class DevelopmentSqliteMigrator(
|
||||
ON "ClassroomReservations" ("ReviewedByUserId");
|
||||
"""
|
||||
];
|
||||
|
||||
private static readonly string[] BackgroundJobOutboxStatements =
|
||||
[
|
||||
"""
|
||||
CREATE TABLE "BackgroundJobOutboxMessages" (
|
||||
"Id" TEXT NOT NULL CONSTRAINT "PK_BackgroundJobOutboxMessages" PRIMARY KEY,
|
||||
"JobKind" INTEGER NOT NULL,
|
||||
"JobId" TEXT NOT NULL,
|
||||
"State" INTEGER NOT NULL,
|
||||
"PublishAttempts" INTEGER NOT NULL,
|
||||
"ProcessingAttempts" INTEGER NOT NULL,
|
||||
"PublishedAt" TEXT NULL,
|
||||
"ProcessingToken" TEXT NULL,
|
||||
"LeaseExpiresAt" TEXT NULL,
|
||||
"CompletedAt" TEXT NULL,
|
||||
"LastError" TEXT NULL,
|
||||
"CreatedAt" TEXT NOT NULL,
|
||||
"UpdatedAt" TEXT NOT NULL
|
||||
);
|
||||
""",
|
||||
"""
|
||||
CREATE UNIQUE INDEX "IX_BackgroundJobOutboxMessages_JobKind_JobId"
|
||||
ON "BackgroundJobOutboxMessages" ("JobKind", "JobId");
|
||||
""",
|
||||
"""
|
||||
CREATE INDEX "IX_BackgroundJobOutboxMessages_LeaseExpiresAt"
|
||||
ON "BackgroundJobOutboxMessages" ("LeaseExpiresAt");
|
||||
""",
|
||||
"""
|
||||
CREATE INDEX "IX_BackgroundJobOutboxMessages_State_CreatedAt"
|
||||
ON "BackgroundJobOutboxMessages" ("State", "CreatedAt");
|
||||
"""
|
||||
];
|
||||
}
|
||||
|
||||
+4845
File diff suppressed because it is too large
Load Diff
+62
@@ -0,0 +1,62 @@
|
||||
using System;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace Jiaowu.Api.Infrastructure.Persistence.Migrations.MySql
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class BackgroundJobOutbox : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.CreateTable(
|
||||
name: "BackgroundJobOutboxMessages",
|
||||
columns: table => new
|
||||
{
|
||||
Id = table.Column<Guid>(type: "char(36)", nullable: false),
|
||||
JobKind = table.Column<int>(type: "int", nullable: false),
|
||||
JobId = table.Column<Guid>(type: "char(36)", nullable: false),
|
||||
State = table.Column<int>(type: "int", nullable: false),
|
||||
PublishAttempts = table.Column<int>(type: "int", nullable: false),
|
||||
ProcessingAttempts = table.Column<int>(type: "int", nullable: false),
|
||||
PublishedAt = table.Column<DateTime>(type: "datetime(6)", nullable: true),
|
||||
ProcessingToken = table.Column<Guid>(type: "char(36)", nullable: true),
|
||||
LeaseExpiresAt = table.Column<DateTime>(type: "datetime(6)", nullable: true),
|
||||
CompletedAt = table.Column<DateTime>(type: "datetime(6)", nullable: true),
|
||||
LastError = table.Column<string>(type: "varchar(2000)", maxLength: 2000, nullable: true),
|
||||
CreatedAt = table.Column<DateTime>(type: "datetime(6)", nullable: false),
|
||||
UpdatedAt = table.Column<DateTime>(type: "datetime(6)", nullable: false)
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
table.PrimaryKey("PK_BackgroundJobOutboxMessages", x => x.Id);
|
||||
})
|
||||
.Annotation("MySQL:Charset", "utf8mb4");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "IX_BackgroundJobOutboxMessages_JobKind_JobId",
|
||||
table: "BackgroundJobOutboxMessages",
|
||||
columns: new[] { "JobKind", "JobId" },
|
||||
unique: true);
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "IX_BackgroundJobOutboxMessages_LeaseExpiresAt",
|
||||
table: "BackgroundJobOutboxMessages",
|
||||
column: "LeaseExpiresAt");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "IX_BackgroundJobOutboxMessages_State_CreatedAt",
|
||||
table: "BackgroundJobOutboxMessages",
|
||||
columns: new[] { "State", "CreatedAt" });
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropTable(
|
||||
name: "BackgroundJobOutboxMessages");
|
||||
}
|
||||
}
|
||||
}
|
||||
+55
@@ -3423,6 +3423,61 @@ namespace Jiaowu.Api.Infrastructure.Persistence.Migrations.MySql
|
||||
b.ToTable("AuditLogs");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("Jiaowu.Api.Domain.System.BackgroundJobOutboxMessage", b =>
|
||||
{
|
||||
b.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("char(36)");
|
||||
|
||||
b.Property<DateTime?>("CompletedAt")
|
||||
.HasColumnType("datetime(6)");
|
||||
|
||||
b.Property<DateTime>("CreatedAt")
|
||||
.HasColumnType("datetime(6)");
|
||||
|
||||
b.Property<Guid>("JobId")
|
||||
.HasColumnType("char(36)");
|
||||
|
||||
b.Property<int>("JobKind")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<string>("LastError")
|
||||
.HasMaxLength(2000)
|
||||
.HasColumnType("varchar(2000)");
|
||||
|
||||
b.Property<DateTime?>("LeaseExpiresAt")
|
||||
.HasColumnType("datetime(6)");
|
||||
|
||||
b.Property<int>("ProcessingAttempts")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<Guid?>("ProcessingToken")
|
||||
.HasColumnType("char(36)");
|
||||
|
||||
b.Property<int>("PublishAttempts")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<DateTime?>("PublishedAt")
|
||||
.HasColumnType("datetime(6)");
|
||||
|
||||
b.Property<int>("State")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<DateTime>("UpdatedAt")
|
||||
.HasColumnType("datetime(6)");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("LeaseExpiresAt");
|
||||
|
||||
b.HasIndex("JobKind", "JobId")
|
||||
.IsUnique();
|
||||
|
||||
b.HasIndex("State", "CreatedAt");
|
||||
|
||||
b.ToTable("BackgroundJobOutboxMessages");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<System.Guid>", b =>
|
||||
{
|
||||
b.Property<int>("Id")
|
||||
|
||||
@@ -1,104 +1,11 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Channels;
|
||||
using Jiaowu.Api.Domain.Academic;
|
||||
using Jiaowu.Api.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace Jiaowu.Api.Infrastructure.Scheduling;
|
||||
|
||||
public sealed class AutomaticScheduleJobQueue
|
||||
{
|
||||
private readonly Channel<Guid> _channel = Channel.CreateUnbounded<Guid>(
|
||||
new UnboundedChannelOptions
|
||||
{
|
||||
SingleReader = true,
|
||||
SingleWriter = false
|
||||
});
|
||||
|
||||
public void Enqueue(Guid jobId)
|
||||
{
|
||||
if (!_channel.Writer.TryWrite(jobId))
|
||||
throw new InvalidOperationException("自动排课任务队列当前不可用。");
|
||||
}
|
||||
|
||||
public IAsyncEnumerable<Guid> ReadAllAsync(CancellationToken cancellationToken) =>
|
||||
_channel.Reader.ReadAllAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class AutomaticScheduleJobWorker(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
AutomaticScheduleJobQueue queue,
|
||||
ILogger<AutomaticScheduleJobWorker> logger) : BackgroundService
|
||||
{
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
await RecoverInterruptedJobsAsync(stoppingToken);
|
||||
|
||||
try
|
||||
{
|
||||
await foreach (var jobId in queue.ReadAllAsync(stoppingToken))
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var processor = scope.ServiceProvider
|
||||
.GetRequiredService<AutomaticScheduleJobProcessor>();
|
||||
await processor.ProcessAsync(jobId, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Unexpected failure while dispatching automatic schedule job {JobId}.",
|
||||
jobId);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
logger.LogInformation("Automatic schedule job worker is stopping.");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RecoverInterruptedJobsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
var jobs = await db.AutomaticScheduleJobs
|
||||
.Where(x =>
|
||||
x.Status == AutomaticScheduleJobStatus.Queued ||
|
||||
x.Status == AutomaticScheduleJobStatus.Running)
|
||||
.OrderBy(x => x.CreatedAt)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
foreach (var job in jobs)
|
||||
{
|
||||
job.Status = AutomaticScheduleJobStatus.Queued;
|
||||
job.ActiveSchedulePlanId = job.SchedulePlanId;
|
||||
job.StartedAt = null;
|
||||
job.CompletedAt = null;
|
||||
job.ErrorMessage = null;
|
||||
}
|
||||
|
||||
if (jobs.Count > 0)
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
foreach (var job in jobs)
|
||||
queue.Enqueue(job.Id);
|
||||
|
||||
if (jobs.Count > 0)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Recovered {JobCount} queued or interrupted automatic schedule jobs.",
|
||||
jobs.Count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class AutomaticScheduleJobProcessor(
|
||||
AppDbContext db,
|
||||
AutomaticScheduleGenerator generator,
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
using System.Threading.Channels;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using Jiaowu.Api.Domain.Academic;
|
||||
using Jiaowu.Api.Infrastructure.Caching;
|
||||
@@ -7,100 +6,6 @@ using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace Jiaowu.Api.Infrastructure.Scheduling;
|
||||
|
||||
public sealed class SchedulePublishJobQueue
|
||||
{
|
||||
private readonly Channel<Guid> _channel = Channel.CreateUnbounded<Guid>(
|
||||
new UnboundedChannelOptions
|
||||
{
|
||||
SingleReader = true,
|
||||
SingleWriter = false
|
||||
});
|
||||
|
||||
public void Enqueue(Guid jobId)
|
||||
{
|
||||
if (!_channel.Writer.TryWrite(jobId))
|
||||
throw new InvalidOperationException("课表发布任务队列当前不可用。");
|
||||
}
|
||||
|
||||
public IAsyncEnumerable<Guid> ReadAllAsync(CancellationToken cancellationToken) =>
|
||||
_channel.Reader.ReadAllAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class SchedulePublishJobWorker(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
SchedulePublishJobQueue queue,
|
||||
ILogger<SchedulePublishJobWorker> logger) : BackgroundService
|
||||
{
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
await RecoverInterruptedJobsAsync(stoppingToken);
|
||||
|
||||
try
|
||||
{
|
||||
await foreach (var jobId in queue.ReadAllAsync(stoppingToken))
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var processor = scope.ServiceProvider
|
||||
.GetRequiredService<SchedulePublishJobProcessor>();
|
||||
await processor.ProcessAsync(jobId, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(
|
||||
exception,
|
||||
"Unexpected failure while dispatching schedule publish job {JobId}.",
|
||||
jobId);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
logger.LogInformation("Schedule publish job worker is stopping.");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RecoverInterruptedJobsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
var jobs = await db.SchedulePublishJobs
|
||||
.Where(x =>
|
||||
x.Status == SchedulePublishJobStatus.Queued ||
|
||||
x.Status == SchedulePublishJobStatus.Running)
|
||||
.OrderBy(x => x.CreatedAt)
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
foreach (var job in jobs)
|
||||
{
|
||||
job.Status = SchedulePublishJobStatus.Queued;
|
||||
job.ActiveAcademicTermId = job.AcademicTermId;
|
||||
job.CompletedSteps = 0;
|
||||
job.CurrentStep = "等待后台检查";
|
||||
job.StartedAt = null;
|
||||
job.CompletedAt = null;
|
||||
job.ErrorMessage = null;
|
||||
}
|
||||
|
||||
if (jobs.Count > 0)
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
foreach (var job in jobs)
|
||||
queue.Enqueue(job.Id);
|
||||
|
||||
if (jobs.Count > 0)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Recovered {JobCount} queued or interrupted schedule publish jobs.",
|
||||
jobs.Count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class SchedulePublishJobProcessor(
|
||||
AppDbContext db,
|
||||
SchedulePlanPublisher publisher,
|
||||
|
||||
Reference in New Issue
Block a user