biss
3ec0f117d3
队列异步执行，拆分查询消除笛卡尔积。
修改的文件（共 13 个）
┌───────────────────────────────────────────────────────────────┬──────────────────────────────────────────────────┐
│ 文件 │ 变更 │
├───────────────────────────────────────────────────────────────┼──────────────────────────────────────────────────┤
│ Domain/Academic/ExamEntities.cs │ 新增 ExamPublishJob 实体 + ExamPublishJobStatus │
│ │ / ExamPublishJobKind 枚举 │
├───────────────────────────────────────────────────────────────┼──────────────────────────────────────────────────┤
│ Domain/System/BackgroundJobOutboxMessage.cs │ BackgroundJobKind 新增 ExamPublish = 6 │
├───────────────────────────────────────────────────────────────┼──────────────────────────────────────────────────┤
│ Infrastructure/BackgroundJobs/BackgroundJobOptions.cs │ 新增 ExamPublishConcurrency 配置项 │
├───────────────────────────────────────────────────────────────┼──────────────────────────────────────────────────┤
│ Infrastructure/BackgroundJobs/BackgroundJobRunner.cs │ RunAsync 和 MarkJobRetryLimitExceeded 添加 │
│ │ ExamPublish 分支 │
├───────────────────────────────────────────────────────────────┼──────────────────────────────────────────────────┤
│ Infrastructure/BackgroundJobs/RabbitMqBackgroundJobs.cs │ JobKinds 数组和 RoutingKey 添加 ExamPublish → │
│ │ "exam.publish" │
├───────────────────────────────────────────────────────────────┼──────────────────────────────────────────────────┤
│ Infrastructure/BackgroundJobs/BackgroundJobOutboxPublisher.cs │ 启动恢复逻辑添加 ExamPublishJobs │
├───────────────────────────────────────────────────────────────┼──────────────────────────────────────────────────┤
│ Infrastructure/Persistence/AppDbContext.cs │ 新增 ExamPublishJobs DbSet │
├───────────────────────────────────────────────────────────────┼──────────────────────────────────────────────────┤
│ Controllers/OperationsController.cs │ CountFailedJobsAsync / GetFailedJobs │
│ │ 添加考试发布失败统计和筛选 │
├───────────────────────────────────────────────────────────────┼──────────────────────────────────────────────────┤
│ Program.cs │ 校验 ExamPublishConcurrency + 注册 │
│ │ ExamPublishJobProcessor │
├───────────────────────────────────────────────────────────────┼──────────────────────────────────────────────────┤
│ Infrastructure/Exams/ExamPublishJobs.cs │ 新文件 — │
│ │ ExamPublishJobProcessor，拆分查询校验后发布 │
├───────────────────────────────────────────────────────────────┼──────────────────────────────────────────────────┤
│ Controllers/ExamsController.cs │ Publish 改为创建后台任务 + 202 返回；新增 GET │
│ │ publish-jobs/{id} / GET plans/{id}/publish-job │
├───────────────────────────────────────────────────────────────┼──────────────────────────────────────────────────┤
│ Controllers/MakeupExamsController.cs │ 同上改造 │
├───────────────────────────────────────────────────────────────┼──────────────────────────────────────────────────┤
│ tests/.../TeachingWorkflowRosterTests.cs │ 更新测试适配新的异步发布模式 │
└───────────────────────────────────────────────────────────────┴──────────────────────────────────────────────────┘
笛卡尔积消除
之前：一个 Include 链拉全部 → EF Core 生成 Sessions × Invigilators × RoomLinks × Seats 笛卡尔积
之后：
- 场次计数：db.ExamSessions.CountAsync（无 JOIN）
- 场次摘要：Select new { Id, ClassroomId, InvigilatorCount, RoomLinkCount }（只查所需列）
- 容量超限：db.ExamRooms.Select(r => new { SeatCount = r.Seats.Count, Capacity })（单表 JOIN）
- 课程冲突：db.ExamRoomSessions.Where(link => ...CourseId != link.ExamRoom!.CourseId)（独立查询）
- 每个查询只做自己需要的 JOIN，互不干扰
测试结果
213 通过，0 失败，0 跳过
配置方式
- BackgroundJobs__Transport=RabbitMq → 走 RabbitMQ 队列 jiaowu.background-jobs.exam.publish
- BackgroundJobs__Transport=InMemory（默认） → 走内存 Channel
- BackgroundJobs__ExamPublishConcurrency=1（默认，可调 1-16）
398 lines
16 KiB
C#
398 lines
16 KiB
C#
using System.Diagnostics;
|
|
using Jiaowu.Api.Domain.Academic;
|
|
using Jiaowu.Api.Domain.System;
|
|
using Jiaowu.Api.Infrastructure.Exams;
|
|
using Jiaowu.Api.Infrastructure.Persistence;
|
|
using Jiaowu.Api.Infrastructure.Scheduling;
|
|
using Microsoft.EntityFrameworkCore;
|
|
|
|
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
|
|
|
public sealed record BackgroundJobRunResult(
|
|
BackgroundJobRunOutcome Outcome,
|
|
TimeSpan RetryAfter)
|
|
{
|
|
public static BackgroundJobRunResult Completed { get; } =
|
|
new(BackgroundJobRunOutcome.Completed, TimeSpan.Zero);
|
|
|
|
public static BackgroundJobRunResult Retry(TimeSpan retryAfter) =>
|
|
new(BackgroundJobRunOutcome.Retry, retryAfter);
|
|
}
|
|
|
|
public enum BackgroundJobRunOutcome
|
|
{
|
|
Completed,
|
|
Retry
|
|
}
|
|
|
|
public sealed class BackgroundJobRunner(
|
|
IServiceScopeFactory scopeFactory,
|
|
IBackgroundJobTransport transport,
|
|
BackgroundJobOptions options,
|
|
BackgroundJobTelemetry telemetry,
|
|
ILogger<BackgroundJobRunner> logger)
|
|
{
|
|
public async Task<BackgroundJobRunResult> RunAsync(
|
|
BackgroundJobEnvelope message,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var startedAt = Stopwatch.GetTimestamp();
|
|
var token = Guid.NewGuid();
|
|
var claim = await TryClaimAsync(message, token, cancellationToken);
|
|
if (!claim.Claimed)
|
|
{
|
|
return claim.Completed
|
|
? BackgroundJobRunResult.Completed
|
|
: BackgroundJobRunResult.Retry(claim.RetryAfter);
|
|
}
|
|
|
|
if (claim.AttemptsExceeded)
|
|
{
|
|
await MarkJobRetryLimitExceededAsync(message, cancellationToken);
|
|
await MarkCompletedAsync(message.OutboxMessageId, token,
|
|
cancellationToken);
|
|
telemetry.RecordProcessing(
|
|
message.JobKind,
|
|
"retry-limit",
|
|
Stopwatch.GetElapsedTime(startedAt));
|
|
return BackgroundJobRunResult.Completed;
|
|
}
|
|
|
|
using var heartbeatCancellation =
|
|
CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
|
var heartbeat = RunHeartbeatAsync(
|
|
message.OutboxMessageId,
|
|
token,
|
|
heartbeatCancellation.Token);
|
|
|
|
try
|
|
{
|
|
await using var scope = scopeFactory.CreateAsyncScope();
|
|
switch (message.JobKind)
|
|
{
|
|
case BackgroundJobKind.AutomaticSchedule:
|
|
await scope.ServiceProvider
|
|
.GetRequiredService<AutomaticScheduleJobProcessor>()
|
|
.ProcessAsync(message.JobId, cancellationToken);
|
|
break;
|
|
case BackgroundJobKind.SchedulePublish:
|
|
await scope.ServiceProvider
|
|
.GetRequiredService<SchedulePublishJobProcessor>()
|
|
.ProcessAsync(message.JobId, cancellationToken);
|
|
break;
|
|
case BackgroundJobKind.MakeupExamAuto:
|
|
await scope.ServiceProvider
|
|
.GetRequiredService<MakeupExamAutoJobProcessor>()
|
|
.ProcessAsync(message.JobId, cancellationToken);
|
|
break;
|
|
case BackgroundJobKind.ExamArrangement:
|
|
await scope.ServiceProvider
|
|
.GetRequiredService<ExamArrangementJobProcessor>()
|
|
.ProcessAsync(message.JobId, cancellationToken);
|
|
break;
|
|
case BackgroundJobKind.ExamSignInExport:
|
|
await scope.ServiceProvider
|
|
.GetRequiredService<ExamSignInExportJobProcessor>()
|
|
.ProcessAsync(message.JobId, cancellationToken);
|
|
break;
|
|
case BackgroundJobKind.ExamPublish:
|
|
await scope.ServiceProvider
|
|
.GetRequiredService<ExamPublishJobProcessor>()
|
|
.ProcessAsync(message.JobId, cancellationToken);
|
|
break;
|
|
default:
|
|
throw new InvalidOperationException(
|
|
$"Unsupported background job kind '{message.JobKind}'.");
|
|
}
|
|
|
|
await MarkCompletedAsync(message.OutboxMessageId, token,
|
|
cancellationToken);
|
|
telemetry.RecordProcessing(
|
|
message.JobKind,
|
|
"completed",
|
|
Stopwatch.GetElapsedTime(startedAt));
|
|
return BackgroundJobRunResult.Completed;
|
|
}
|
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
|
{
|
|
throw;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
logger.LogError(
|
|
exception,
|
|
"Background job {JobKind}/{JobId} will be retried.",
|
|
message.JobKind,
|
|
message.JobId);
|
|
await ReleaseForRetryAsync(
|
|
message.OutboxMessageId,
|
|
token,
|
|
exception,
|
|
CancellationToken.None);
|
|
telemetry.RecordProcessing(
|
|
message.JobKind,
|
|
"retry",
|
|
Stopwatch.GetElapsedTime(startedAt));
|
|
return BackgroundJobRunResult.Retry(TimeSpan.FromSeconds(2));
|
|
}
|
|
finally
|
|
{
|
|
await heartbeatCancellation.CancelAsync();
|
|
try
|
|
{
|
|
await heartbeat;
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
// Expected when processing completes or the application stops.
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task<(
|
|
bool Claimed,
|
|
bool Completed,
|
|
bool AttemptsExceeded,
|
|
TimeSpan RetryAfter)>
|
|
TryClaimAsync(
|
|
BackgroundJobEnvelope message,
|
|
Guid token,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
await using var scope = scopeFactory.CreateAsyncScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
|
var now = DateTime.UtcNow;
|
|
var leaseExpiresAt = now.AddSeconds(options.LeaseSeconds);
|
|
var claimed = await db.BackgroundJobOutboxMessages
|
|
.Where(x =>
|
|
x.Id == message.OutboxMessageId &&
|
|
x.JobKind == message.JobKind &&
|
|
x.JobId == message.JobId &&
|
|
(x.State == BackgroundJobOutboxState.Pending ||
|
|
x.State == BackgroundJobOutboxState.Publishing ||
|
|
x.State == BackgroundJobOutboxState.Published ||
|
|
(x.State == BackgroundJobOutboxState.Processing &&
|
|
x.LeaseExpiresAt != null &&
|
|
x.LeaseExpiresAt < now)))
|
|
.ExecuteUpdateAsync(
|
|
setters => setters
|
|
.SetProperty(x => x.State, BackgroundJobOutboxState.Processing)
|
|
.SetProperty(x => x.ProcessingToken, token)
|
|
.SetProperty(x => x.LeaseExpiresAt, leaseExpiresAt)
|
|
.SetProperty(
|
|
x => x.ProcessingAttempts,
|
|
x => x.ProcessingAttempts + 1)
|
|
.SetProperty(x => x.LastError, (string?)null),
|
|
cancellationToken);
|
|
if (claimed == 1)
|
|
{
|
|
var attempts = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
|
.Where(x => x.Id == message.OutboxMessageId)
|
|
.Select(x => x.ProcessingAttempts)
|
|
.SingleAsync(cancellationToken);
|
|
return (
|
|
true,
|
|
false,
|
|
attempts > options.ProcessingAttemptLimit,
|
|
TimeSpan.Zero);
|
|
}
|
|
|
|
var current = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
|
.Where(x => x.Id == message.OutboxMessageId)
|
|
.Select(x => new { x.State, x.LeaseExpiresAt })
|
|
.FirstOrDefaultAsync(cancellationToken);
|
|
if (current is null || current.State == BackgroundJobOutboxState.Completed)
|
|
return (false, true, false, TimeSpan.Zero);
|
|
|
|
var retryAfter = current.LeaseExpiresAt.HasValue
|
|
? current.LeaseExpiresAt.Value - now
|
|
: TimeSpan.FromSeconds(2);
|
|
retryAfter = TimeSpan.FromSeconds(Math.Clamp(
|
|
retryAfter.TotalSeconds,
|
|
1,
|
|
options.LeaseSeconds));
|
|
return (false, false, false, retryAfter);
|
|
}
|
|
|
|
private async Task MarkJobRetryLimitExceededAsync(
|
|
BackgroundJobEnvelope message,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
await using var scope = scopeFactory.CreateAsyncScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
|
var completedAt = DateTime.UtcNow;
|
|
var error = $"后台任务连续处理失败超过 {options.ProcessingAttemptLimit} 次，" +
|
|
"已停止自动重试，请检查服务日志后重新创建任务。";
|
|
|
|
switch (message.JobKind)
|
|
{
|
|
case BackgroundJobKind.AutomaticSchedule:
|
|
await db.AutomaticScheduleJobs
|
|
.Where(x =>
|
|
x.Id == message.JobId &&
|
|
x.Status != AutomaticScheduleJobStatus.Succeeded &&
|
|
x.Status != AutomaticScheduleJobStatus.Failed)
|
|
.ExecuteUpdateAsync(
|
|
setters => setters
|
|
.SetProperty(
|
|
x => x.Status,
|
|
AutomaticScheduleJobStatus.Failed)
|
|
.SetProperty(x => x.ActiveSchedulePlanId, (Guid?)null)
|
|
.SetProperty(x => x.ErrorMessage, error)
|
|
.SetProperty(x => x.CompletedAt, completedAt),
|
|
cancellationToken);
|
|
break;
|
|
case BackgroundJobKind.SchedulePublish:
|
|
await db.SchedulePublishJobs
|
|
.Where(x =>
|
|
x.Id == message.JobId &&
|
|
x.Status != SchedulePublishJobStatus.Succeeded &&
|
|
x.Status != SchedulePublishJobStatus.Failed)
|
|
.ExecuteUpdateAsync(
|
|
setters => setters
|
|
.SetProperty(
|
|
x => x.Status,
|
|
SchedulePublishJobStatus.Failed)
|
|
.SetProperty(x => x.ActiveAcademicTermId, (Guid?)null)
|
|
.SetProperty(x => x.CurrentStep, "后台处理已停止")
|
|
.SetProperty(x => x.ErrorMessage, error)
|
|
.SetProperty(x => x.CompletedAt, completedAt),
|
|
cancellationToken);
|
|
break;
|
|
case BackgroundJobKind.MakeupExamAuto:
|
|
await db.MakeupExamAutoJobs
|
|
.Where(x =>
|
|
x.Id == message.JobId &&
|
|
x.Status != MakeupExamAutoJobStatus.Succeeded &&
|
|
x.Status != MakeupExamAutoJobStatus.Failed)
|
|
.ExecuteUpdateAsync(
|
|
setters => setters
|
|
.SetProperty(
|
|
x => x.Status,
|
|
MakeupExamAutoJobStatus.Failed)
|
|
.SetProperty(x => x.ErrorMessage, error)
|
|
.SetProperty(x => x.CompletedAt, completedAt),
|
|
cancellationToken);
|
|
break;
|
|
case BackgroundJobKind.ExamArrangement:
|
|
await db.ExamArrangementJobs
|
|
.Where(x =>
|
|
x.Id == message.JobId &&
|
|
x.Status != ExamArrangementJobStatus.Succeeded &&
|
|
x.Status != ExamArrangementJobStatus.Failed)
|
|
.ExecuteUpdateAsync(
|
|
setters => setters
|
|
.SetProperty(
|
|
x => x.Status,
|
|
ExamArrangementJobStatus.Failed)
|
|
.SetProperty(x => x.ActivePlanId, (Guid?)null)
|
|
.SetProperty(x => x.CurrentStep, "后台处理已停止")
|
|
.SetProperty(x => x.ErrorMessage, error)
|
|
.SetProperty(x => x.CompletedAt, completedAt),
|
|
cancellationToken);
|
|
break;
|
|
case BackgroundJobKind.ExamPublish:
|
|
await db.ExamPublishJobs
|
|
.Where(x =>
|
|
x.Id == message.JobId &&
|
|
x.Status != ExamPublishJobStatus.Succeeded &&
|
|
x.Status != ExamPublishJobStatus.Failed)
|
|
.ExecuteUpdateAsync(
|
|
setters => setters
|
|
.SetProperty(
|
|
x => x.Status,
|
|
ExamPublishJobStatus.Failed)
|
|
.SetProperty(x => x.ActivePlanId, (Guid?)null)
|
|
.SetProperty(x => x.CurrentStep, "后台处理已停止")
|
|
.SetProperty(x => x.ErrorMessage, error)
|
|
.SetProperty(x => x.CompletedAt, completedAt),
|
|
cancellationToken);
|
|
break;
|
|
default:
|
|
throw new ArgumentOutOfRangeException(
|
|
nameof(message.JobKind),
|
|
message.JobKind,
|
|
null);
|
|
}
|
|
|
|
logger.LogError(
|
|
"Background job {JobKind}/{JobId} exceeded {AttemptLimit} processing attempts.",
|
|
message.JobKind,
|
|
message.JobId,
|
|
options.ProcessingAttemptLimit);
|
|
}
|
|
|
|
private async Task RunHeartbeatAsync(
|
|
Guid outboxMessageId,
|
|
Guid token,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var intervalSeconds = Math.Max(5, options.LeaseSeconds / 3);
|
|
while (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
await Task.Delay(TimeSpan.FromSeconds(intervalSeconds), cancellationToken);
|
|
await using var scope = scopeFactory.CreateAsyncScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
|
await db.BackgroundJobOutboxMessages
|
|
.Where(x =>
|
|
x.Id == outboxMessageId &&
|
|
x.State == BackgroundJobOutboxState.Processing &&
|
|
x.ProcessingToken == token)
|
|
.ExecuteUpdateAsync(
|
|
setters => setters.SetProperty(
|
|
x => x.LeaseExpiresAt,
|
|
DateTime.UtcNow.AddSeconds(options.LeaseSeconds)),
|
|
cancellationToken);
|
|
}
|
|
}
|
|
|
|
private async Task MarkCompletedAsync(
|
|
Guid outboxMessageId,
|
|
Guid token,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
await using var scope = scopeFactory.CreateAsyncScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
|
await db.BackgroundJobOutboxMessages
|
|
.Where(x =>
|
|
x.Id == outboxMessageId &&
|
|
x.State == BackgroundJobOutboxState.Processing &&
|
|
x.ProcessingToken == token)
|
|
.ExecuteUpdateAsync(
|
|
setters => setters
|
|
.SetProperty(x => x.State, BackgroundJobOutboxState.Completed)
|
|
.SetProperty(x => x.CompletedAt, DateTime.UtcNow)
|
|
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
|
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null),
|
|
cancellationToken);
|
|
}
|
|
|
|
private async Task ReleaseForRetryAsync(
|
|
Guid outboxMessageId,
|
|
Guid token,
|
|
Exception exception,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
await using var scope = scopeFactory.CreateAsyncScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
|
var message = exception.GetBaseException().Message;
|
|
if (message.Length > 2000)
|
|
message = message[..2000];
|
|
var state = transport.IsDurable
|
|
? BackgroundJobOutboxState.Published
|
|
: BackgroundJobOutboxState.Pending;
|
|
await db.BackgroundJobOutboxMessages
|
|
.Where(x =>
|
|
x.Id == outboxMessageId &&
|
|
x.State == BackgroundJobOutboxState.Processing &&
|
|
x.ProcessingToken == token)
|
|
.ExecuteUpdateAsync(
|
|
setters => setters
|
|
.SetProperty(x => x.State, state)
|
|
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
|
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null)
|
|
.SetProperty(x => x.LastError, message),
|
|
cancellationToken);
|
|
}
|
|
}
|