RabbitMQ 默认从单一回调通道提升为三类任务独立并行,跨类型并发能力由 1 提升到 3;每类还能独立配置 1–16 个消费者。[RabbitMqBackgroundJobs.cs (line 143)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/RabbitMqBackgroundJobs.cs:143) InMemory 开发模式同步改为按任务类型隔离队列,避免某类长任务堵塞其他任务。 租约恢复检查从“每发布一条执行一次”改为默认每 60 秒维护一次。 启动恢复改为数据库 NOT EXISTS 查询,不再把全部历史 Outbox 加载进内存。[BackgroundJobOutboxPublisher.cs (line 9)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/BackgroundJobOutboxPublisher.cs:9) 完成消息默认保留 14 天,之后按每批 500 条清理,并增加对应组合索引。 /health/messaging 现在返回各状态积压量、过期租约和最老任务等待时间。[BackgroundJobMonitoringService.cs (line 16)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/BackgroundJobMonitoringService.cs:16) 新增 Jiaowu.BackgroundJobs 运行时指标,覆盖发布量、处理量、发布耗时、处理耗时和清理量。[BackgroundJobTelemetry.cs (line 9)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/BackgroundJobTelemetry.cs:9) 配置、Compose 和调优建议已更新。[README.md (line 257)](E:/jiaowu/README.md:257)
266 lines
11 KiB
C#
266 lines
11 KiB
C#
using Jiaowu.Api.Domain.Academic;
|
|
using Jiaowu.Api.Domain.System;
|
|
using Jiaowu.Api.Infrastructure.BackgroundJobs;
|
|
using Jiaowu.Api.Infrastructure.Persistence;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
|
|
namespace Jiaowu.Api.Tests;
|
|
|
|
public sealed class BackgroundJobOutboxTests
|
|
{
|
|
[Fact]
|
|
public async Task In_memory_transport_keeps_job_types_on_independent_channels()
|
|
{
|
|
var transport = new InMemoryBackgroundJobTransport();
|
|
var automatic = new BackgroundJobEnvelope(
|
|
Guid.NewGuid(),
|
|
BackgroundJobKind.AutomaticSchedule,
|
|
Guid.NewGuid());
|
|
var publish = new BackgroundJobEnvelope(
|
|
Guid.NewGuid(),
|
|
BackgroundJobKind.SchedulePublish,
|
|
Guid.NewGuid());
|
|
await transport.PublishAsync(automatic, CancellationToken.None);
|
|
await transport.PublishAsync(publish, CancellationToken.None);
|
|
|
|
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(1));
|
|
await using var automaticReader = transport
|
|
.ReadAllAsync(BackgroundJobKind.AutomaticSchedule, timeout.Token)
|
|
.GetAsyncEnumerator(timeout.Token);
|
|
await using var publishReader = transport
|
|
.ReadAllAsync(BackgroundJobKind.SchedulePublish, timeout.Token)
|
|
.GetAsyncEnumerator(timeout.Token);
|
|
|
|
Assert.True(await automaticReader.MoveNextAsync());
|
|
Assert.Equal(automatic, automaticReader.Current);
|
|
Assert.True(await publishReader.MoveNextAsync());
|
|
Assert.Equal(publish, publishReader.Current);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Publisher_recovers_pending_message_and_cleans_old_completion()
|
|
{
|
|
var databasePath = Path.Combine(
|
|
Path.GetTempPath(),
|
|
$"jiaowu-outbox-{Guid.NewGuid():N}.sqlite");
|
|
var services = new ServiceCollection();
|
|
services.AddLogging();
|
|
services.AddDbContext<AppDbContext>(options =>
|
|
options.UseSqlite($"Data Source={databasePath};Pooling=False"));
|
|
var options = new BackgroundJobOptions
|
|
{
|
|
PollIntervalMilliseconds = 100,
|
|
CompletedRetentionDays = 1,
|
|
CleanupBatchSize = 10
|
|
};
|
|
services.AddSingleton(options);
|
|
services.AddSingleton<InMemoryBackgroundJobTransport>();
|
|
services.AddSingleton<IBackgroundJobTransport>(provider =>
|
|
provider.GetRequiredService<InMemoryBackgroundJobTransport>());
|
|
services.AddSingleton<BackgroundJobTelemetry>();
|
|
services.AddSingleton<BackgroundJobOutboxPublisher>();
|
|
var provider = services.BuildServiceProvider();
|
|
|
|
try
|
|
{
|
|
Guid jobId;
|
|
Guid oldCompletedOutboxId;
|
|
await using (var scope = provider.CreateAsyncScope())
|
|
{
|
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
|
await db.Database.EnsureCreatedAsync();
|
|
var term = new AcademicTerm
|
|
{
|
|
Code = "RECOVERY",
|
|
Name = "补投测试学期",
|
|
AcademicYear = "2026-2027",
|
|
Season = TermSeason.Autumn,
|
|
StartDate = new DateOnly(2026, 9, 1),
|
|
EndDate = new DateOnly(2027, 1, 15)
|
|
};
|
|
var plan = new SchedulePlan
|
|
{
|
|
AcademicTerm = term,
|
|
Name = "补投测试排课",
|
|
Version = "V1"
|
|
};
|
|
var job = new AutomaticScheduleJob
|
|
{
|
|
SchedulePlan = plan,
|
|
ActiveSchedulePlanId = plan.Id
|
|
};
|
|
var oldCompletedOutbox = BackgroundJobOutboxMessage.Create(
|
|
BackgroundJobKind.SchedulePublish,
|
|
Guid.NewGuid());
|
|
oldCompletedOutbox.State = BackgroundJobOutboxState.Completed;
|
|
oldCompletedOutbox.CompletedAt = DateTime.UtcNow.AddDays(-2);
|
|
oldCompletedOutboxId = oldCompletedOutbox.Id;
|
|
jobId = job.Id;
|
|
db.AddRange(term, plan, job, oldCompletedOutbox);
|
|
await db.SaveChangesAsync();
|
|
}
|
|
|
|
var publisher = provider.GetRequiredService<BackgroundJobOutboxPublisher>();
|
|
await publisher.StartAsync(CancellationToken.None);
|
|
try
|
|
{
|
|
var published = false;
|
|
for (var attempt = 0; attempt < 50 && !published; attempt++)
|
|
{
|
|
await Task.Delay(50);
|
|
await using var scope = provider.CreateAsyncScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
|
published = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
|
.AnyAsync(x =>
|
|
x.JobId == jobId &&
|
|
x.JobKind == BackgroundJobKind.AutomaticSchedule &&
|
|
x.State == BackgroundJobOutboxState.Published) &&
|
|
!await db.BackgroundJobOutboxMessages.AsNoTracking()
|
|
.AnyAsync(x => x.Id == oldCompletedOutboxId);
|
|
}
|
|
|
|
Assert.True(published);
|
|
}
|
|
finally
|
|
{
|
|
await publisher.StopAsync(CancellationToken.None);
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
await provider.DisposeAsync();
|
|
File.Delete(databasePath);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Monitoring_snapshot_reports_backlog_and_expired_leases()
|
|
{
|
|
var databasePath = Path.Combine(
|
|
Path.GetTempPath(),
|
|
$"jiaowu-monitoring-{Guid.NewGuid():N}.sqlite");
|
|
var options = new DbContextOptionsBuilder<AppDbContext>()
|
|
.UseSqlite($"Data Source={databasePath};Pooling=False")
|
|
.Options;
|
|
|
|
try
|
|
{
|
|
await using var db = new AppDbContext(options);
|
|
await db.Database.EnsureCreatedAsync();
|
|
var pending = BackgroundJobOutboxMessage.Create(
|
|
BackgroundJobKind.AutomaticSchedule,
|
|
Guid.NewGuid());
|
|
pending.CreatedAt = DateTime.UtcNow.AddMinutes(-2);
|
|
var processing = BackgroundJobOutboxMessage.Create(
|
|
BackgroundJobKind.MakeupExamAuto,
|
|
Guid.NewGuid());
|
|
processing.State = BackgroundJobOutboxState.Processing;
|
|
processing.LeaseExpiresAt = DateTime.UtcNow.AddMinutes(-1);
|
|
db.AddRange(pending, processing);
|
|
await db.SaveChangesAsync();
|
|
|
|
var snapshot = await new BackgroundJobMonitoringService(db)
|
|
.GetSnapshotAsync(CancellationToken.None);
|
|
|
|
Assert.Equal(1, snapshot.Pending);
|
|
Assert.Equal(1, snapshot.Processing);
|
|
Assert.Equal(1, snapshot.ExpiredLeases);
|
|
Assert.NotNull(snapshot.OldestUnfinishedAt);
|
|
Assert.True(snapshot.OldestUnfinishedAgeSeconds >= 100);
|
|
}
|
|
finally
|
|
{
|
|
File.Delete(databasePath);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Runner_stops_after_retry_limit_and_ignores_duplicate_delivery()
|
|
{
|
|
var databasePath = Path.Combine(
|
|
Path.GetTempPath(),
|
|
$"jiaowu-runner-{Guid.NewGuid():N}.sqlite");
|
|
var services = new ServiceCollection();
|
|
services.AddLogging();
|
|
services.AddDbContext<AppDbContext>(options =>
|
|
options.UseSqlite($"Data Source={databasePath};Pooling=False"));
|
|
services.AddScoped<Jiaowu.Api.Infrastructure.Scheduling.AutomaticScheduleGenerator>();
|
|
services.AddScoped<Jiaowu.Api.Infrastructure.Scheduling.AutomaticScheduleJobProcessor>();
|
|
services.AddSingleton(new BackgroundJobOptions());
|
|
services.AddSingleton<InMemoryBackgroundJobTransport>();
|
|
services.AddSingleton<IBackgroundJobTransport>(provider =>
|
|
provider.GetRequiredService<InMemoryBackgroundJobTransport>());
|
|
services.AddSingleton<BackgroundJobTelemetry>();
|
|
services.AddSingleton<BackgroundJobRunner>();
|
|
var provider = services.BuildServiceProvider();
|
|
|
|
try
|
|
{
|
|
BackgroundJobEnvelope envelope;
|
|
await using (var scope = provider.CreateAsyncScope())
|
|
{
|
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
|
await db.Database.EnsureCreatedAsync();
|
|
var term = new AcademicTerm
|
|
{
|
|
Code = "OUTBOX",
|
|
Name = "消息测试学期",
|
|
AcademicYear = "2026-2027",
|
|
Season = TermSeason.Autumn,
|
|
StartDate = new DateOnly(2026, 9, 1),
|
|
EndDate = new DateOnly(2027, 1, 15)
|
|
};
|
|
var plan = new SchedulePlan
|
|
{
|
|
AcademicTerm = term,
|
|
Name = "消息测试排课",
|
|
Version = "V1"
|
|
};
|
|
var job = new AutomaticScheduleJob
|
|
{
|
|
SchedulePlan = plan,
|
|
ActiveSchedulePlanId = plan.Id,
|
|
Status = AutomaticScheduleJobStatus.Queued
|
|
};
|
|
var outbox = BackgroundJobOutboxMessage.Create(
|
|
BackgroundJobKind.AutomaticSchedule,
|
|
job.Id);
|
|
outbox.State = BackgroundJobOutboxState.Published;
|
|
outbox.ProcessingAttempts = 5;
|
|
db.AddRange(term, plan, job, outbox);
|
|
await db.SaveChangesAsync();
|
|
envelope = new BackgroundJobEnvelope(
|
|
outbox.Id,
|
|
outbox.JobKind,
|
|
outbox.JobId);
|
|
}
|
|
|
|
var runner = provider.GetRequiredService<BackgroundJobRunner>();
|
|
var first = await runner.RunAsync(envelope, CancellationToken.None);
|
|
var duplicate = await runner.RunAsync(envelope, CancellationToken.None);
|
|
|
|
Assert.Equal(BackgroundJobRunOutcome.Completed, first.Outcome);
|
|
Assert.Equal(BackgroundJobRunOutcome.Completed, duplicate.Outcome);
|
|
await using var assertScope = provider.CreateAsyncScope();
|
|
var assertDb = assertScope.ServiceProvider.GetRequiredService<AppDbContext>();
|
|
var persistedOutbox =
|
|
await assertDb.BackgroundJobOutboxMessages.SingleAsync();
|
|
Assert.Equal(BackgroundJobOutboxState.Completed, persistedOutbox.State);
|
|
Assert.NotNull(persistedOutbox.CompletedAt);
|
|
Assert.Null(persistedOutbox.ProcessingToken);
|
|
Assert.Null(persistedOutbox.LeaseExpiresAt);
|
|
Assert.Equal(6, persistedOutbox.ProcessingAttempts);
|
|
var persistedJob = await assertDb.AutomaticScheduleJobs.SingleAsync();
|
|
Assert.Equal(AutomaticScheduleJobStatus.Failed, persistedJob.Status);
|
|
Assert.Null(persistedJob.ActiveSchedulePlanId);
|
|
Assert.Contains("超过 5 次", persistedJob.ErrorMessage);
|
|
}
|
|
finally
|
|
{
|
|
await provider.DisposeAsync();
|
|
File.Delete(databasePath);
|
|
}
|
|
}
|
|
}
|