第二阶段优化已完成,重点是提高后台任务吞吐并降低 Outbox 数据库开销。

RabbitMQ 默认从单一回调通道提升为三类任务独立并行,跨类型并发能力由 1 提升到 3;每类还能独立配置 1–16 个消费者。[RabbitMqBackgroundJobs.cs (line 143)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/RabbitMqBackgroundJobs.cs:143)
InMemory 开发模式同步改为按任务类型隔离队列,避免某类长任务堵塞其他任务。
租约恢复检查从“每发布一条执行一次”改为默认每 60 秒维护一次。
启动恢复改为数据库 NOT EXISTS 查询,不再把全部历史 Outbox 加载进内存。[BackgroundJobOutboxPublisher.cs (line 9)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/BackgroundJobOutboxPublisher.cs:9)
完成消息默认保留 14 天,之后按每批 500 条清理,并增加对应组合索引。
/health/messaging 现在返回各状态积压量、过期租约和最老任务等待时间。[BackgroundJobMonitoringService.cs (line 16)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/BackgroundJobMonitoringService.cs:16)
新增 Jiaowu.BackgroundJobs 运行时指标,覆盖发布量、处理量、发布耗时、处理耗时和清理量。[BackgroundJobTelemetry.cs (line 9)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/BackgroundJobTelemetry.cs:9)
配置、Compose 和调优建议已更新。[README.md (line 257)](E:/jiaowu/README.md:257)
This commit is contained in:
2026-07-26 21:05:18 +08:00 Unverified
parent 6300645120
commit 034f1eacb8
21 changed files with 579 additions and 135 deletions
+3
View File
@@ -10,6 +10,9 @@ MYSQL_ROOT_PASSWORD=
RABBITMQ_USER=jiaowu RABBITMQ_USER=jiaowu
RABBITMQ_PASSWORD= RABBITMQ_PASSWORD=
BACKGROUND_JOB_AUTOMATIC_SCHEDULE_CONCURRENCY=1
BACKGROUND_JOB_SCHEDULE_PUBLISH_CONCURRENCY=1
BACKGROUND_JOB_MAKEUP_EXAM_AUTO_CONCURRENCY=1
JWT_KEY= JWT_KEY=
ALLOWED_HOSTS=localhost ALLOWED_HOSTS=localhost
+3
View File
@@ -11,6 +11,9 @@ ConnectionStrings__MySql="Server=db.example.edu.cn;Port=3306;Database=jiaowu;Use
# 单机可继续使用 InMemory;多实例生产部署建议启用 RabbitMQ。 # 单机可继续使用 InMemory;多实例生产部署建议启用 RabbitMQ。
BackgroundJobs__Transport=InMemory BackgroundJobs__Transport=InMemory
BackgroundJobs__AutomaticScheduleConcurrency=1
BackgroundJobs__SchedulePublishConcurrency=1
BackgroundJobs__MakeupExamAutoConcurrency=1
# RabbitMq__HostName=rabbitmq.example.edu.cn # RabbitMq__HostName=rabbitmq.example.edu.cn
# RabbitMq__Port=5671 # RabbitMq__Port=5671
# RabbitMq__UserName=jiaowu # RabbitMq__UserName=jiaowu
+17 -4
View File
@@ -224,7 +224,8 @@ SQLite 只用于本地开发:新库通过 `EnsureCreated` 建立,已有开
- `/health/cache`:检查可选 Redis;未配置 Redis 时返回 `disabled`Redis - `/health/cache`:检查可选 Redis;未配置 Redis 时返回 `disabled`Redis
故障不会影响数据库就绪探针。 故障不会影响数据库就绪探针。
- `/health/messaging`:检查后台任务传输;单机内存队列返回 `memory`,启用 - `/health/messaging`:检查后台任务传输;单机内存队列返回 `memory`,启用
RabbitMQ 时实际检查代理连接 RabbitMQ 时实际检查代理连接,同时返回各 Outbox 状态的积压数、过期租约数和
最老未完成任务的等待时间。
### 查询缓存与 Redis ### 查询缓存与 Redis
@@ -270,6 +271,9 @@ BackgroundJobs__Transport=InMemory
```text ```text
BackgroundJobs__Transport=RabbitMq BackgroundJobs__Transport=RabbitMq
BackgroundJobs__AutomaticScheduleConcurrency=1
BackgroundJobs__SchedulePublishConcurrency=1
BackgroundJobs__MakeupExamAutoConcurrency=1
RabbitMq__HostName=rabbitmq.example.edu.cn RabbitMq__HostName=rabbitmq.example.edu.cn
RabbitMq__Port=5671 RabbitMq__Port=5671
RabbitMq__UserName=jiaowu RabbitMq__UserName=jiaowu
@@ -280,9 +284,18 @@ RabbitMq__TlsServerName=rabbitmq.example.edu.cn
``` ```
RabbitMQ 传输使用持久消息、发布确认、手动消费确认、每种任务独立队列和死信队列。 RabbitMQ 传输使用持久消息、发布确认、手动消费确认、每种任务独立队列和死信队列。
默认创建 Quorum Queue,重任务的消费者预取数为 1。MySQL 或 RabbitMQ 暂时不可用时 默认创建 Quorum Queue,重任务的消费者预取数为 1。三类任务各自至少有一个消费者
未完成消息会根据 Outbox 状态和租约继续补投,消费者必须保持业务处理幂等。迁移服务 因此不同类型的任务不会再互相阻塞;单类任务的并发度可独立设置为 1-16。自动排课
应先应用 `BackgroundJobOutbox` 数据库迁移,再启动应用实例。 通常最消耗 CPU,建议先保持为 1,再根据 CPU、数据库连接池和任务等待时间逐级调到
2 或 3;不要只提高预取数。
Outbox 租约恢复改为按维护周期执行,避免积压发布时每条消息都额外扫描数据库;已完成
消息默认保留 14 天并按每批 500 条清理,可使用 `CompletedRetentionDays`
`MaintenanceIntervalSeconds``CleanupBatchSize` 调整。应用暴露
`Jiaowu.BackgroundJobs` Meter,其中包含发布量、处理量、发布耗时、处理耗时和清理量,
可接入现有 OpenTelemetry/运行时指标采集器。MySQL 或 RabbitMQ 暂时不可用时,未完成
消息会根据 Outbox 状态和租约继续补投。迁移服务应先应用
`BackgroundJobOutbox` 数据库迁移,再启动应用实例。
## 跨平台发布与 Docker ## 跨平台发布与 Docker
+3
View File
@@ -17,6 +17,9 @@ x-jiaowu-environment: &jiaowu-environment
Cache__Enabled: "true" Cache__Enabled: "true"
Cache__KeyPrefix: "jiaowu:v1" Cache__KeyPrefix: "jiaowu:v1"
BackgroundJobs__Transport: RabbitMq BackgroundJobs__Transport: RabbitMq
BackgroundJobs__AutomaticScheduleConcurrency: "${BACKGROUND_JOB_AUTOMATIC_SCHEDULE_CONCURRENCY:-1}"
BackgroundJobs__SchedulePublishConcurrency: "${BACKGROUND_JOB_SCHEDULE_PUBLISH_CONCURRENCY:-1}"
BackgroundJobs__MakeupExamAutoConcurrency: "${BACKGROUND_JOB_MAKEUP_EXAM_AUTO_CONCURRENCY:-1}"
RabbitMq__HostName: rabbitmq RabbitMq__HostName: rabbitmq
RabbitMq__Port: "5672" RabbitMq__Port: "5672"
RabbitMq__UserName: "${RABBITMQ_USER:-jiaowu}" RabbitMq__UserName: "${RABBITMQ_USER:-jiaowu}"
@@ -0,0 +1,56 @@
using Jiaowu.Api.Domain.System;
using Jiaowu.Api.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
public sealed record BackgroundJobBacklogSnapshot(
int Pending,
int Publishing,
int Published,
int Processing,
int ExpiredLeases,
DateTime? OldestUnfinishedAt,
double? OldestUnfinishedAgeSeconds);
public sealed class BackgroundJobMonitoringService(AppDbContext db)
{
public async Task<BackgroundJobBacklogSnapshot> GetSnapshotAsync(
CancellationToken cancellationToken)
{
var counts = await db.BackgroundJobOutboxMessages.AsNoTracking()
.Where(x => x.State != BackgroundJobOutboxState.Completed)
.GroupBy(x => x.State)
.Select(group => new { State = group.Key, Count = group.Count() })
.ToListAsync(cancellationToken);
var countByState = counts.ToDictionary(x => x.State, x => x.Count);
var now = DateTime.UtcNow;
var expiredLeases = await db.BackgroundJobOutboxMessages.AsNoTracking()
.CountAsync(
x => (x.State == BackgroundJobOutboxState.Publishing ||
x.State == BackgroundJobOutboxState.Processing) &&
x.LeaseExpiresAt != null &&
x.LeaseExpiresAt < now,
cancellationToken);
var oldestUnfinishedAt = await db.BackgroundJobOutboxMessages
.AsNoTracking()
.Where(x => x.State != BackgroundJobOutboxState.Completed)
.Select(x => (DateTime?)x.CreatedAt)
.MinAsync(cancellationToken);
double? ageSeconds = oldestUnfinishedAt.HasValue
? Math.Max(0, (now - oldestUnfinishedAt.Value).TotalSeconds)
: null;
return new BackgroundJobBacklogSnapshot(
GetCount(BackgroundJobOutboxState.Pending),
GetCount(BackgroundJobOutboxState.Publishing),
GetCount(BackgroundJobOutboxState.Published),
GetCount(BackgroundJobOutboxState.Processing),
expiredLeases,
oldestUnfinishedAt,
ageSeconds);
int GetCount(BackgroundJobOutboxState state) =>
countByState.GetValueOrDefault(state);
}
}
@@ -1,3 +1,5 @@
using Jiaowu.Api.Domain.System;
namespace Jiaowu.Api.Infrastructure.BackgroundJobs; namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
public sealed class BackgroundJobOptions public sealed class BackgroundJobOptions
@@ -8,13 +10,27 @@ public sealed class BackgroundJobOptions
public int PollIntervalMilliseconds { get; set; } = 500; public int PollIntervalMilliseconds { get; set; } = 500;
public int LeaseSeconds { get; set; } = 120; public int LeaseSeconds { get; set; } = 120;
public ushort PrefetchCount { get; set; } = 1; public ushort PrefetchCount { get; set; } = 1;
public int AutomaticScheduleConcurrency { get; set; } = 1;
public int SchedulePublishConcurrency { get; set; } = 1;
public int MakeupExamAutoConcurrency { get; set; } = 1;
public string Exchange { get; set; } = "jiaowu.background-jobs"; public string Exchange { get; set; } = "jiaowu.background-jobs";
public string QueuePrefix { get; set; } = "jiaowu.background-jobs"; public string QueuePrefix { get; set; } = "jiaowu.background-jobs";
public bool UseQuorumQueues { get; set; } = true; public bool UseQuorumQueues { get; set; } = true;
public int ProcessingAttemptLimit { get; set; } = 5; public int ProcessingAttemptLimit { get; set; } = 5;
public int MaintenanceIntervalSeconds { get; set; } = 60;
public int CompletedRetentionDays { get; set; } = 14;
public int CleanupBatchSize { get; set; } = 500;
public bool UsesRabbitMq => public bool UsesRabbitMq =>
Transport.Equals("RabbitMq", StringComparison.OrdinalIgnoreCase); Transport.Equals("RabbitMq", StringComparison.OrdinalIgnoreCase);
public int ConsumerConcurrency(BackgroundJobKind kind) => kind switch
{
BackgroundJobKind.AutomaticSchedule => AutomaticScheduleConcurrency,
BackgroundJobKind.SchedulePublish => SchedulePublishConcurrency,
BackgroundJobKind.MakeupExamAuto => MakeupExamAutoConcurrency,
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null)
};
} }
public sealed class RabbitMqOptions public sealed class RabbitMqOptions
@@ -1,3 +1,4 @@
using System.Diagnostics;
using Jiaowu.Api.Domain.Academic; using Jiaowu.Api.Domain.Academic;
using Jiaowu.Api.Domain.System; using Jiaowu.Api.Domain.System;
using Jiaowu.Api.Infrastructure.Persistence; using Jiaowu.Api.Infrastructure.Persistence;
@@ -9,17 +10,26 @@ public sealed class BackgroundJobOutboxPublisher(
IServiceScopeFactory scopeFactory, IServiceScopeFactory scopeFactory,
IBackgroundJobTransport transport, IBackgroundJobTransport transport,
BackgroundJobOptions options, BackgroundJobOptions options,
BackgroundJobTelemetry telemetry,
ILogger<BackgroundJobOutboxPublisher> logger) : BackgroundService ILogger<BackgroundJobOutboxPublisher> logger) : BackgroundService
{ {
protected override async Task ExecuteAsync(CancellationToken stoppingToken) protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{ {
await PrepareForStartupAsync(stoppingToken); await PrepareForStartupAsync(stoppingToken);
var nextMaintenanceAt = DateTime.MinValue;
while (!stoppingToken.IsCancellationRequested) while (!stoppingToken.IsCancellationRequested)
{ {
try try
{
if (DateTime.UtcNow >= nextMaintenanceAt)
{ {
await RecoverExpiredProcessingAsync(stoppingToken); await RecoverExpiredProcessingAsync(stoppingToken);
await CleanupCompletedAsync(stoppingToken);
nextMaintenanceAt = DateTime.UtcNow.AddSeconds(
options.MaintenanceIntervalSeconds);
}
var claimed = await ClaimNextAsync(stoppingToken); var claimed = await ClaimNextAsync(stoppingToken);
if (claimed is null) if (claimed is null)
{ {
@@ -62,51 +72,107 @@ public sealed class BackgroundJobOutboxPublisher(
await AddMissingOutboxMessagesAsync(db, cancellationToken); await AddMissingOutboxMessagesAsync(db, cancellationToken);
} }
private static async Task AddMissingOutboxMessagesAsync( private async Task AddMissingOutboxMessagesAsync(
AppDbContext db, AppDbContext db,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
var existing = await db.BackgroundJobOutboxMessages.AsNoTracking()
.Select(x => new { x.JobKind, x.JobId })
.ToListAsync(cancellationToken);
var existingKeys = existing
.Select(x => (x.JobKind, x.JobId))
.ToHashSet();
var automaticJobs = await db.AutomaticScheduleJobs.AsNoTracking() var automaticJobs = await db.AutomaticScheduleJobs.AsNoTracking()
.Where(x => x.Status == AutomaticScheduleJobStatus.Queued || .Where(x =>
x.Status == AutomaticScheduleJobStatus.Running) (x.Status == AutomaticScheduleJobStatus.Queued ||
x.Status == AutomaticScheduleJobStatus.Running) &&
!db.BackgroundJobOutboxMessages.Any(message =>
message.JobKind == BackgroundJobKind.AutomaticSchedule &&
message.JobId == x.Id))
.Select(x => x.Id) .Select(x => x.Id)
.ToListAsync(cancellationToken); .ToListAsync(cancellationToken);
var publishJobs = await db.SchedulePublishJobs.AsNoTracking() var publishJobs = await db.SchedulePublishJobs.AsNoTracking()
.Where(x => x.Status == SchedulePublishJobStatus.Queued || .Where(x =>
x.Status == SchedulePublishJobStatus.Running) (x.Status == SchedulePublishJobStatus.Queued ||
x.Status == SchedulePublishJobStatus.Running) &&
!db.BackgroundJobOutboxMessages.Any(message =>
message.JobKind == BackgroundJobKind.SchedulePublish &&
message.JobId == x.Id))
.Select(x => x.Id) .Select(x => x.Id)
.ToListAsync(cancellationToken); .ToListAsync(cancellationToken);
var makeupJobs = await db.MakeupExamAutoJobs.AsNoTracking() var makeupJobs = await db.MakeupExamAutoJobs.AsNoTracking()
.Where(x => x.Status == MakeupExamAutoJobStatus.Queued || .Where(x =>
x.Status == MakeupExamAutoJobStatus.Running) (x.Status == MakeupExamAutoJobStatus.Queued ||
x.Status == MakeupExamAutoJobStatus.Running) &&
!db.BackgroundJobOutboxMessages.Any(message =>
message.JobKind == BackgroundJobKind.MakeupExamAuto &&
message.JobId == x.Id))
.Select(x => x.Id) .Select(x => x.Id)
.ToListAsync(cancellationToken); .ToListAsync(cancellationToken);
AddMissing(BackgroundJobKind.AutomaticSchedule, automaticJobs); var missingKeys = automaticJobs
AddMissing(BackgroundJobKind.SchedulePublish, publishJobs); .Select(id => (BackgroundJobKind.AutomaticSchedule, id))
AddMissing(BackgroundJobKind.MakeupExamAuto, makeupJobs); .Concat(publishJobs.Select(id =>
(BackgroundJobKind.SchedulePublish, id)))
if (db.ChangeTracker.HasChanges()) .Concat(makeupJobs.Select(id =>
await db.SaveChangesAsync(cancellationToken); (BackgroundJobKind.MakeupExamAuto, id)))
.ToList();
void AddMissing(BackgroundJobKind kind, IEnumerable<Guid> jobIds) foreach (var (kind, jobId) in missingKeys)
{
foreach (var jobId in jobIds)
{
if (existingKeys.Add((kind, jobId)))
{ {
db.BackgroundJobOutboxMessages.Add( db.BackgroundJobOutboxMessages.Add(
BackgroundJobOutboxMessage.Create(kind, jobId)); BackgroundJobOutboxMessage.Create(kind, jobId));
} }
if (!db.ChangeTracker.HasChanges())
return;
try
{
await db.SaveChangesAsync(cancellationToken);
}
catch (DbUpdateException)
{
db.ChangeTracker.Clear();
foreach (var (kind, jobId) in missingKeys)
{
var recovered = await db.BackgroundJobOutboxMessages
.AsNoTracking()
.AnyAsync(
message => message.JobKind == kind &&
message.JobId == jobId,
cancellationToken);
if (!recovered)
throw;
}
logger.LogInformation(
"Another application instance recovered {Count} missing outbox messages first.",
missingKeys.Count);
} }
} }
private async Task CleanupCompletedAsync(CancellationToken cancellationToken)
{
await using var scope = scopeFactory.CreateAsyncScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var cutoff = DateTime.UtcNow.AddDays(-options.CompletedRetentionDays);
var completedIds = await db.BackgroundJobOutboxMessages.AsNoTracking()
.Where(x =>
x.State == BackgroundJobOutboxState.Completed &&
x.CompletedAt != null &&
x.CompletedAt < cutoff)
.OrderBy(x => x.CompletedAt)
.Select(x => x.Id)
.Take(options.CleanupBatchSize)
.ToListAsync(cancellationToken);
if (completedIds.Count == 0)
return;
foreach (var id in completedIds)
{
db.BackgroundJobOutboxMessages.Remove(
new BackgroundJobOutboxMessage { Id = id });
}
await db.SaveChangesAsync(cancellationToken);
telemetry.RecordCleaned(completedIds.Count);
logger.LogInformation(
"Removed {Count} completed background job outbox messages older than {Cutoff}.",
completedIds.Count,
cutoff);
} }
private async Task RecoverExpiredProcessingAsync( private async Task RecoverExpiredProcessingAsync(
@@ -175,6 +241,7 @@ public sealed class BackgroundJobOutboxPublisher(
Guid token, Guid token,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
var startedAt = Stopwatch.GetTimestamp();
try try
{ {
await transport.PublishAsync(message, cancellationToken); await transport.PublishAsync(message, cancellationToken);
@@ -193,6 +260,11 @@ public sealed class BackgroundJobOutboxPublisher(
.SetProperty(x => x.ProcessingToken, (Guid?)null) .SetProperty(x => x.ProcessingToken, (Guid?)null)
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null), .SetProperty(x => x.LeaseExpiresAt, (DateTime?)null),
cancellationToken); cancellationToken);
telemetry.RecordPublish(
message.JobKind,
transport.IsDurable,
"published",
Stopwatch.GetElapsedTime(startedAt));
} }
catch (Exception exception) catch (Exception exception)
{ {
@@ -213,6 +285,11 @@ public sealed class BackgroundJobOutboxPublisher(
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null) .SetProperty(x => x.LeaseExpiresAt, (DateTime?)null)
.SetProperty(x => x.LastError, messageText), .SetProperty(x => x.LastError, messageText),
cancellationToken); cancellationToken);
telemetry.RecordPublish(
message.JobKind,
transport.IsDurable,
"failed",
Stopwatch.GetElapsedTime(startedAt));
throw; throw;
} }
} }
@@ -1,3 +1,4 @@
using System.Diagnostics;
using Jiaowu.Api.Domain.Academic; using Jiaowu.Api.Domain.Academic;
using Jiaowu.Api.Domain.System; using Jiaowu.Api.Domain.System;
using Jiaowu.Api.Infrastructure.Exams; using Jiaowu.Api.Infrastructure.Exams;
@@ -28,12 +29,14 @@ public sealed class BackgroundJobRunner(
IServiceScopeFactory scopeFactory, IServiceScopeFactory scopeFactory,
IBackgroundJobTransport transport, IBackgroundJobTransport transport,
BackgroundJobOptions options, BackgroundJobOptions options,
BackgroundJobTelemetry telemetry,
ILogger<BackgroundJobRunner> logger) ILogger<BackgroundJobRunner> logger)
{ {
public async Task<BackgroundJobRunResult> RunAsync( public async Task<BackgroundJobRunResult> RunAsync(
BackgroundJobEnvelope message, BackgroundJobEnvelope message,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
var startedAt = Stopwatch.GetTimestamp();
var token = Guid.NewGuid(); var token = Guid.NewGuid();
var claim = await TryClaimAsync(message, token, cancellationToken); var claim = await TryClaimAsync(message, token, cancellationToken);
if (!claim.Claimed) if (!claim.Claimed)
@@ -48,6 +51,10 @@ public sealed class BackgroundJobRunner(
await MarkJobRetryLimitExceededAsync(message, cancellationToken); await MarkJobRetryLimitExceededAsync(message, cancellationToken);
await MarkCompletedAsync(message.OutboxMessageId, token, await MarkCompletedAsync(message.OutboxMessageId, token,
cancellationToken); cancellationToken);
telemetry.RecordProcessing(
message.JobKind,
"retry-limit",
Stopwatch.GetElapsedTime(startedAt));
return BackgroundJobRunResult.Completed; return BackgroundJobRunResult.Completed;
} }
@@ -85,6 +92,10 @@ public sealed class BackgroundJobRunner(
await MarkCompletedAsync(message.OutboxMessageId, token, await MarkCompletedAsync(message.OutboxMessageId, token,
cancellationToken); cancellationToken);
telemetry.RecordProcessing(
message.JobKind,
"completed",
Stopwatch.GetElapsedTime(startedAt));
return BackgroundJobRunResult.Completed; return BackgroundJobRunResult.Completed;
} }
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
@@ -103,6 +114,10 @@ public sealed class BackgroundJobRunner(
token, token,
exception, exception,
CancellationToken.None); CancellationToken.None);
telemetry.RecordProcessing(
message.JobKind,
"retry",
Stopwatch.GetElapsedTime(startedAt));
return BackgroundJobRunResult.Retry(TimeSpan.FromSeconds(2)); return BackgroundJobRunResult.Retry(TimeSpan.FromSeconds(2));
} }
finally finally
@@ -0,0 +1,74 @@
using System.Diagnostics;
using System.Diagnostics.Metrics;
using Jiaowu.Api.Domain.System;
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
public sealed class BackgroundJobTelemetry : IDisposable
{
public const string MeterName = "Jiaowu.BackgroundJobs";
private readonly Meter _meter = new(MeterName, "1.0.0");
private readonly Counter<long> _published;
private readonly Counter<long> _processed;
private readonly Histogram<double> _publishDuration;
private readonly Histogram<double> _processingDuration;
private readonly Counter<long> _cleaned;
public BackgroundJobTelemetry()
{
_published = _meter.CreateCounter<long>(
"jiaowu.background_jobs.published",
unit: "{message}");
_processed = _meter.CreateCounter<long>(
"jiaowu.background_jobs.processed",
unit: "{job}");
_publishDuration = _meter.CreateHistogram<double>(
"jiaowu.background_jobs.publish.duration",
unit: "ms");
_processingDuration = _meter.CreateHistogram<double>(
"jiaowu.background_jobs.processing.duration",
unit: "ms");
_cleaned = _meter.CreateCounter<long>(
"jiaowu.background_jobs.outbox.cleaned",
unit: "{message}");
}
public void RecordPublish(
BackgroundJobKind kind,
bool durable,
string outcome,
TimeSpan duration)
{
var tags = new TagList
{
{ "job.kind", kind.ToString() },
{ "messaging.backend", durable ? "rabbitmq" : "memory" },
{ "job.outcome", outcome }
};
_published.Add(1, tags);
_publishDuration.Record(duration.TotalMilliseconds, tags);
}
public void RecordProcessing(
BackgroundJobKind kind,
string outcome,
TimeSpan duration)
{
var tags = new TagList
{
{ "job.kind", kind.ToString() },
{ "job.outcome", outcome }
};
_processed.Add(1, tags);
_processingDuration.Record(duration.TotalMilliseconds, tags);
}
public void RecordCleaned(int count)
{
if (count > 0)
_cleaned.Add(count);
}
public void Dispose() => _meter.Dispose();
}
@@ -21,26 +21,31 @@ public interface IBackgroundJobTransport
public sealed class InMemoryBackgroundJobTransport : IBackgroundJobTransport public sealed class InMemoryBackgroundJobTransport : IBackgroundJobTransport
{ {
private readonly Channel<BackgroundJobEnvelope> _channel = private readonly IReadOnlyDictionary<
Channel.CreateBounded<BackgroundJobEnvelope>( BackgroundJobKind,
Channel<BackgroundJobEnvelope>> _channels =
Enum.GetValues<BackgroundJobKind>().ToDictionary(
kind => kind,
_ => Channel.CreateBounded<BackgroundJobEnvelope>(
new BoundedChannelOptions(256) new BoundedChannelOptions(256)
{ {
FullMode = BoundedChannelFullMode.Wait, FullMode = BoundedChannelFullMode.Wait,
SingleReader = true, SingleReader = false,
SingleWriter = false SingleWriter = false
}); }));
public bool IsDurable => false; public bool IsDurable => false;
public ValueTask PublishAsync( public ValueTask PublishAsync(
BackgroundJobEnvelope message, BackgroundJobEnvelope message,
CancellationToken cancellationToken) => CancellationToken cancellationToken) =>
_channel.Writer.WriteAsync(message, cancellationToken); _channels[message.JobKind].Writer.WriteAsync(message, cancellationToken);
public Task<bool> CheckHealthAsync(CancellationToken cancellationToken) => public Task<bool> CheckHealthAsync(CancellationToken cancellationToken) =>
Task.FromResult(true); Task.FromResult(true);
public IAsyncEnumerable<BackgroundJobEnvelope> ReadAllAsync( public IAsyncEnumerable<BackgroundJobEnvelope> ReadAllAsync(
BackgroundJobKind kind,
CancellationToken cancellationToken) => CancellationToken cancellationToken) =>
_channel.Reader.ReadAllAsync(cancellationToken); _channels[kind].Reader.ReadAllAsync(cancellationToken);
} }
@@ -1,27 +1,46 @@
using Jiaowu.Api.Domain.System;
namespace Jiaowu.Api.Infrastructure.BackgroundJobs; namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
public sealed class InMemoryBackgroundJobWorker( public sealed class InMemoryBackgroundJobWorker(
InMemoryBackgroundJobTransport transport, InMemoryBackgroundJobTransport transport,
BackgroundJobRunner runner, BackgroundJobRunner runner,
BackgroundJobOptions options,
ILogger<InMemoryBackgroundJobWorker> logger) : BackgroundService ILogger<InMemoryBackgroundJobWorker> logger) : BackgroundService
{ {
protected override async Task ExecuteAsync(CancellationToken stoppingToken) protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{ {
var workers = Enum.GetValues<BackgroundJobKind>()
.SelectMany(kind => Enumerable.Range(
0,
options.ConsumerConcurrency(kind))
.Select(_ => ConsumeAsync(kind, stoppingToken)))
.ToArray();
try try
{ {
await foreach (var message in transport.ReadAllAsync(stoppingToken)) await Task.WhenAll(workers);
{
var result = await runner.RunAsync(message, stoppingToken);
if (result.Outcome != BackgroundJobRunOutcome.Retry)
continue;
await Task.Delay(result.RetryAfter, stoppingToken);
await transport.PublishAsync(message, stoppingToken);
}
} }
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{ {
logger.LogInformation("In-memory background job worker is stopping."); logger.LogInformation("In-memory background job worker is stopping.");
} }
} }
private async Task ConsumeAsync(
BackgroundJobKind kind,
CancellationToken cancellationToken)
{
await foreach (var message in transport.ReadAllAsync(
kind,
cancellationToken))
{
var result = await runner.RunAsync(message, cancellationToken);
if (result.Outcome != BackgroundJobRunOutcome.Retry)
continue;
await Task.Delay(result.RetryAfter, cancellationToken);
await transport.PublishAsync(message, cancellationToken);
}
}
} }
@@ -82,6 +82,7 @@ public sealed class RabbitMqBackgroundJobTransport(
_connection = await RabbitMqBackgroundJobTopology.CreateConnectionAsync( _connection = await RabbitMqBackgroundJobTopology.CreateConnectionAsync(
rabbitOptions, rabbitOptions,
"jiaowu-background-job-publisher", "jiaowu-background-job-publisher",
consumerDispatchConcurrency: 1,
cancellationToken); cancellationToken);
_channel = await _connection.CreateChannelAsync( _channel = await _connection.CreateChannelAsync(
new CreateChannelOptions( new CreateChannelOptions(
@@ -153,12 +154,19 @@ public sealed class RabbitMqBackgroundJobWorker(
var channels = new List<IChannel>(); var channels = new List<IChannel>();
try try
{ {
var consumerCount = RabbitMqBackgroundJobTopology.JobKinds.Sum(
jobOptions.ConsumerConcurrency);
connection = await RabbitMqBackgroundJobTopology.CreateConnectionAsync( connection = await RabbitMqBackgroundJobTopology.CreateConnectionAsync(
rabbitOptions, rabbitOptions,
"jiaowu-background-job-worker", "jiaowu-background-job-worker",
(ushort)consumerCount,
stoppingToken); stoppingToken);
foreach (var kind in RabbitMqBackgroundJobTopology.JobKinds) foreach (var kind in RabbitMqBackgroundJobTopology.JobKinds)
{
for (var workerIndex = 0;
workerIndex < jobOptions.ConsumerConcurrency(kind);
workerIndex++)
{ {
var channel = await connection.CreateChannelAsync( var channel = await connection.CreateChannelAsync(
new CreateChannelOptions( new CreateChannelOptions(
@@ -186,7 +194,8 @@ public sealed class RabbitMqBackgroundJobWorker(
var deliveryToken = deliveryCancellation.Token; var deliveryToken = deliveryCancellation.Token;
try try
{ {
var message = JsonSerializer.Deserialize<BackgroundJobEnvelope>( var message = JsonSerializer
.Deserialize<BackgroundJobEnvelope>(
eventArgs.Body.Span); eventArgs.Body.Span);
if (message is null || message.JobKind != kind) if (message is null || message.JobKind != kind)
{ {
@@ -201,7 +210,8 @@ public sealed class RabbitMqBackgroundJobWorker(
var result = await runner.RunAsync( var result = await runner.RunAsync(
message, message,
deliveryToken); deliveryToken);
if (result.Outcome == BackgroundJobRunOutcome.Completed) if (result.Outcome ==
BackgroundJobRunOutcome.Completed)
{ {
await channel.BasicAckAsync( await channel.BasicAckAsync(
eventArgs.DeliveryTag, eventArgs.DeliveryTag,
@@ -229,25 +239,29 @@ public sealed class RabbitMqBackgroundJobWorker(
exception, exception,
"RabbitMQ delivery for {JobKind} failed and will be requeued.", "RabbitMQ delivery for {JobKind} failed and will be requeued.",
kind); kind);
if (channel.IsOpen) if (!channel.IsOpen)
{ return;
await channel.BasicNackAsync( await channel.BasicNackAsync(
eventArgs.DeliveryTag, eventArgs.DeliveryTag,
multiple: false, multiple: false,
requeue: true, requeue: true,
CancellationToken.None); CancellationToken.None);
} }
}
}; };
await channel.BasicConsumeAsync( await channel.BasicConsumeAsync(
RabbitMqBackgroundJobTopology.QueueName(jobOptions, kind), RabbitMqBackgroundJobTopology.QueueName(
jobOptions,
kind),
autoAck: false, autoAck: false,
consumer, consumer,
stoppingToken); stoppingToken);
} }
}
logger.LogInformation( logger.LogInformation(
"RabbitMQ background job consumers are connected to {HostName}:{Port}.", "{ConsumerCount} RabbitMQ background job consumers are connected to {HostName}:{Port}.",
consumerCount,
rabbitOptions.HostName, rabbitOptions.HostName,
rabbitOptions.Port); rabbitOptions.Port);
while (connection.IsOpen && !stoppingToken.IsCancellationRequested) while (connection.IsOpen && !stoppingToken.IsCancellationRequested)
@@ -308,6 +322,7 @@ internal static class RabbitMqBackgroundJobTopology
public static async Task<IConnection> CreateConnectionAsync( public static async Task<IConnection> CreateConnectionAsync(
RabbitMqOptions options, RabbitMqOptions options,
string clientName, string clientName,
ushort consumerDispatchConcurrency,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
var factory = new ConnectionFactory var factory = new ConnectionFactory
@@ -321,7 +336,7 @@ internal static class RabbitMqBackgroundJobTopology
AutomaticRecoveryEnabled = false, AutomaticRecoveryEnabled = false,
TopologyRecoveryEnabled = false, TopologyRecoveryEnabled = false,
RequestedHeartbeat = TimeSpan.FromSeconds(30), RequestedHeartbeat = TimeSpan.FromSeconds(30),
ConsumerDispatchConcurrency = 1 ConsumerDispatchConcurrency = consumerDispatchConcurrency
}; };
if (options.UseTls) if (options.UseTls)
{ {
@@ -1000,6 +1000,7 @@ public sealed class AppDbContext(DbContextOptions<AppDbContext> options)
entity.Property(x => x.LastError).HasMaxLength(2000); entity.Property(x => x.LastError).HasMaxLength(2000);
entity.HasIndex(x => new { x.JobKind, x.JobId }).IsUnique(); entity.HasIndex(x => new { x.JobKind, x.JobId }).IsUnique();
entity.HasIndex(x => new { x.State, x.CreatedAt }); entity.HasIndex(x => new { x.State, x.CreatedAt });
entity.HasIndex(x => new { x.State, x.CompletedAt });
entity.HasIndex(x => x.LeaseExpiresAt); entity.HasIndex(x => x.LeaseExpiresAt);
}); });
@@ -2050,6 +2050,10 @@ public sealed class DevelopmentSqliteMigrator(
""" """
CREATE INDEX "IX_BackgroundJobOutboxMessages_State_CreatedAt" CREATE INDEX "IX_BackgroundJobOutboxMessages_State_CreatedAt"
ON "BackgroundJobOutboxMessages" ("State", "CreatedAt"); ON "BackgroundJobOutboxMessages" ("State", "CreatedAt");
""",
"""
CREATE INDEX "IX_BackgroundJobOutboxMessages_State_CompletedAt"
ON "BackgroundJobOutboxMessages" ("State", "CompletedAt");
""" """
]; ];
} }
@@ -3478,6 +3478,8 @@ namespace Jiaowu.Api.Infrastructure.Persistence.Migrations.MySql
b.HasIndex("State", "CreatedAt"); b.HasIndex("State", "CreatedAt");
b.HasIndex("State", "CompletedAt");
b.ToTable("BackgroundJobOutboxMessages"); b.ToTable("BackgroundJobOutboxMessages");
}); });
@@ -50,6 +50,11 @@ namespace Jiaowu.Api.Infrastructure.Persistence.Migrations.MySql
name: "IX_BackgroundJobOutboxMessages_State_CreatedAt", name: "IX_BackgroundJobOutboxMessages_State_CreatedAt",
table: "BackgroundJobOutboxMessages", table: "BackgroundJobOutboxMessages",
columns: new[] { "State", "CreatedAt" }); columns: new[] { "State", "CreatedAt" });
migrationBuilder.CreateIndex(
name: "IX_BackgroundJobOutboxMessages_State_CompletedAt",
table: "BackgroundJobOutboxMessages",
columns: new[] { "State", "CompletedAt" });
} }
/// <inheritdoc /> /// <inheritdoc />
@@ -3475,6 +3475,8 @@ namespace Jiaowu.Api.Infrastructure.Persistence.Migrations.MySql
b.HasIndex("State", "CreatedAt"); b.HasIndex("State", "CreatedAt");
b.HasIndex("State", "CompletedAt");
b.ToTable("BackgroundJobOutboxMessages"); b.ToTable("BackgroundJobOutboxMessages");
}); });
+36 -2
View File
@@ -137,7 +137,13 @@ if (!backgroundJobOptions.Transport.Equals(
if (backgroundJobOptions.PollIntervalMilliseconds is < 100 or > 30000 || if (backgroundJobOptions.PollIntervalMilliseconds is < 100 or > 30000 ||
backgroundJobOptions.LeaseSeconds is < 30 or > 3600 || backgroundJobOptions.LeaseSeconds is < 30 or > 3600 ||
backgroundJobOptions.PrefetchCount is < 1 or > 100 || backgroundJobOptions.PrefetchCount is < 1 or > 100 ||
backgroundJobOptions.AutomaticScheduleConcurrency is < 1 or > 16 ||
backgroundJobOptions.SchedulePublishConcurrency is < 1 or > 16 ||
backgroundJobOptions.MakeupExamAutoConcurrency is < 1 or > 16 ||
backgroundJobOptions.ProcessingAttemptLimit is < 1 or > 100 || backgroundJobOptions.ProcessingAttemptLimit is < 1 or > 100 ||
backgroundJobOptions.MaintenanceIntervalSeconds is < 10 or > 3600 ||
backgroundJobOptions.CompletedRetentionDays is < 1 or > 3650 ||
backgroundJobOptions.CleanupBatchSize is < 10 or > 5000 ||
string.IsNullOrWhiteSpace(backgroundJobOptions.Exchange) || string.IsNullOrWhiteSpace(backgroundJobOptions.Exchange) ||
string.IsNullOrWhiteSpace(backgroundJobOptions.QueuePrefix)) string.IsNullOrWhiteSpace(backgroundJobOptions.QueuePrefix))
{ {
@@ -269,6 +275,8 @@ builder.Services.AddScoped<ExamArrangementService>();
builder.Services.AddScoped<MakeupExamEligibilityService>(); builder.Services.AddScoped<MakeupExamEligibilityService>();
builder.Services.AddScoped<MakeupExamArrangementService>(); builder.Services.AddScoped<MakeupExamArrangementService>();
builder.Services.AddScoped<MakeupExamAutoJobProcessor>(); builder.Services.AddScoped<MakeupExamAutoJobProcessor>();
builder.Services.AddSingleton<BackgroundJobTelemetry>();
builder.Services.AddScoped<BackgroundJobMonitoringService>();
builder.Services.AddSingleton<BackgroundJobRunner>(); builder.Services.AddSingleton<BackgroundJobRunner>();
if (backgroundJobOptions.UsesRabbitMq) if (backgroundJobOptions.UsesRabbitMq)
{ {
@@ -534,15 +542,41 @@ static async Task<IResult> CheckCacheHealthAsync(
static async Task<IResult> CheckMessagingHealthAsync( static async Task<IResult> CheckMessagingHealthAsync(
IBackgroundJobTransport transport, IBackgroundJobTransport transport,
BackgroundJobMonitoringService monitoring,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
var healthy = await transport.CheckHealthAsync(cancellationToken); var healthy = await transport.CheckHealthAsync(cancellationToken);
var backend = transport.IsDurable ? "rabbitmq" : "memory"; var backend = transport.IsDurable ? "rabbitmq" : "memory";
try
{
var backlog = await monitoring.GetSnapshotAsync(cancellationToken);
return healthy return healthy
? Results.Ok(new { Status = "healthy", Backend = backend }) ? Results.Ok(new
{
Status = "healthy",
Backend = backend,
Backlog = backlog
})
: Results.Json( : Results.Json(
new { Status = "unhealthy", Backend = backend }, new
{
Status = "unhealthy",
Backend = backend,
Backlog = backlog
},
statusCode: StatusCodes.Status503ServiceUnavailable); statusCode: StatusCodes.Status503ServiceUnavailable);
}
catch
{
return Results.Json(
new
{
Status = "unhealthy",
Backend = backend,
Backlog = (object?)null
},
statusCode: StatusCodes.Status503ServiceUnavailable);
}
} }
public partial class Program; public partial class Program;
+7 -1
View File
@@ -24,10 +24,16 @@
"PollIntervalMilliseconds": 500, "PollIntervalMilliseconds": 500,
"LeaseSeconds": 120, "LeaseSeconds": 120,
"PrefetchCount": 1, "PrefetchCount": 1,
"AutomaticScheduleConcurrency": 1,
"SchedulePublishConcurrency": 1,
"MakeupExamAutoConcurrency": 1,
"Exchange": "jiaowu.background-jobs", "Exchange": "jiaowu.background-jobs",
"QueuePrefix": "jiaowu.background-jobs", "QueuePrefix": "jiaowu.background-jobs",
"UseQuorumQueues": true, "UseQuorumQueues": true,
"ProcessingAttemptLimit": 5 "ProcessingAttemptLimit": 5,
"MaintenanceIntervalSeconds": 60,
"CompletedRetentionDays": 14,
"CleanupBatchSize": 500
}, },
"RabbitMq": { "RabbitMq": {
"HostName": "localhost", "HostName": "localhost",
@@ -30,6 +30,14 @@ public sealed class AutomaticScheduleGeneratorTests
Assert.True(await db.ScheduleTimeSlots.CountAsync() == 0); Assert.True(await db.ScheduleTimeSlots.CountAsync() == 0);
Assert.True(await db.AutomaticScheduleJobs.CountAsync() == 0); Assert.True(await db.AutomaticScheduleJobs.CountAsync() == 0);
Assert.True(await db.BackgroundJobOutboxMessages.CountAsync() == 0); Assert.True(await db.BackgroundJobOutboxMessages.CountAsync() == 0);
Assert.True(await db.Database
.SqlQueryRaw<int>(
"""
SELECT COUNT(*) AS "Value"
FROM pragma_index_list('BackgroundJobOutboxMessages')
WHERE name = 'IX_BackgroundJobOutboxMessages_State_CompletedAt'
""")
.AnyAsync(value => value > 0));
Assert.True(await db.Database Assert.True(await db.Database
.SqlQueryRaw<int>( .SqlQueryRaw<int>(
""" """
@@ -10,7 +10,36 @@ namespace Jiaowu.Api.Tests;
public sealed class BackgroundJobOutboxTests public sealed class BackgroundJobOutboxTests
{ {
[Fact] [Fact]
public async Task Publisher_moves_pending_message_to_in_memory_transport() public async Task In_memory_transport_keeps_job_types_on_independent_channels()
{
var transport = new InMemoryBackgroundJobTransport();
var automatic = new BackgroundJobEnvelope(
Guid.NewGuid(),
BackgroundJobKind.AutomaticSchedule,
Guid.NewGuid());
var publish = new BackgroundJobEnvelope(
Guid.NewGuid(),
BackgroundJobKind.SchedulePublish,
Guid.NewGuid());
await transport.PublishAsync(automatic, CancellationToken.None);
await transport.PublishAsync(publish, CancellationToken.None);
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(1));
await using var automaticReader = transport
.ReadAllAsync(BackgroundJobKind.AutomaticSchedule, timeout.Token)
.GetAsyncEnumerator(timeout.Token);
await using var publishReader = transport
.ReadAllAsync(BackgroundJobKind.SchedulePublish, timeout.Token)
.GetAsyncEnumerator(timeout.Token);
Assert.True(await automaticReader.MoveNextAsync());
Assert.Equal(automatic, automaticReader.Current);
Assert.True(await publishReader.MoveNextAsync());
Assert.Equal(publish, publishReader.Current);
}
[Fact]
public async Task Publisher_recovers_pending_message_and_cleans_old_completion()
{ {
var databasePath = Path.Combine( var databasePath = Path.Combine(
Path.GetTempPath(), Path.GetTempPath(),
@@ -21,18 +50,22 @@ public sealed class BackgroundJobOutboxTests
options.UseSqlite($"Data Source={databasePath};Pooling=False")); options.UseSqlite($"Data Source={databasePath};Pooling=False"));
var options = new BackgroundJobOptions var options = new BackgroundJobOptions
{ {
PollIntervalMilliseconds = 100 PollIntervalMilliseconds = 100,
CompletedRetentionDays = 1,
CleanupBatchSize = 10
}; };
services.AddSingleton(options); services.AddSingleton(options);
services.AddSingleton<InMemoryBackgroundJobTransport>(); services.AddSingleton<InMemoryBackgroundJobTransport>();
services.AddSingleton<IBackgroundJobTransport>(provider => services.AddSingleton<IBackgroundJobTransport>(provider =>
provider.GetRequiredService<InMemoryBackgroundJobTransport>()); provider.GetRequiredService<InMemoryBackgroundJobTransport>());
services.AddSingleton<BackgroundJobTelemetry>();
services.AddSingleton<BackgroundJobOutboxPublisher>(); services.AddSingleton<BackgroundJobOutboxPublisher>();
var provider = services.BuildServiceProvider(); var provider = services.BuildServiceProvider();
try try
{ {
Guid jobId; Guid jobId;
Guid oldCompletedOutboxId;
await using (var scope = provider.CreateAsyncScope()) await using (var scope = provider.CreateAsyncScope())
{ {
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>(); var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
@@ -57,8 +90,14 @@ public sealed class BackgroundJobOutboxTests
SchedulePlan = plan, SchedulePlan = plan,
ActiveSchedulePlanId = plan.Id ActiveSchedulePlanId = plan.Id
}; };
var oldCompletedOutbox = BackgroundJobOutboxMessage.Create(
BackgroundJobKind.SchedulePublish,
Guid.NewGuid());
oldCompletedOutbox.State = BackgroundJobOutboxState.Completed;
oldCompletedOutbox.CompletedAt = DateTime.UtcNow.AddDays(-2);
oldCompletedOutboxId = oldCompletedOutbox.Id;
jobId = job.Id; jobId = job.Id;
db.AddRange(term, plan, job); db.AddRange(term, plan, job, oldCompletedOutbox);
await db.SaveChangesAsync(); await db.SaveChangesAsync();
} }
@@ -76,7 +115,9 @@ public sealed class BackgroundJobOutboxTests
.AnyAsync(x => .AnyAsync(x =>
x.JobId == jobId && x.JobId == jobId &&
x.JobKind == BackgroundJobKind.AutomaticSchedule && x.JobKind == BackgroundJobKind.AutomaticSchedule &&
x.State == BackgroundJobOutboxState.Published); x.State == BackgroundJobOutboxState.Published) &&
!await db.BackgroundJobOutboxMessages.AsNoTracking()
.AnyAsync(x => x.Id == oldCompletedOutboxId);
} }
Assert.True(published); Assert.True(published);
@@ -93,6 +134,47 @@ public sealed class BackgroundJobOutboxTests
} }
} }
[Fact]
public async Task Monitoring_snapshot_reports_backlog_and_expired_leases()
{
var databasePath = Path.Combine(
Path.GetTempPath(),
$"jiaowu-monitoring-{Guid.NewGuid():N}.sqlite");
var options = new DbContextOptionsBuilder<AppDbContext>()
.UseSqlite($"Data Source={databasePath};Pooling=False")
.Options;
try
{
await using var db = new AppDbContext(options);
await db.Database.EnsureCreatedAsync();
var pending = BackgroundJobOutboxMessage.Create(
BackgroundJobKind.AutomaticSchedule,
Guid.NewGuid());
pending.CreatedAt = DateTime.UtcNow.AddMinutes(-2);
var processing = BackgroundJobOutboxMessage.Create(
BackgroundJobKind.MakeupExamAuto,
Guid.NewGuid());
processing.State = BackgroundJobOutboxState.Processing;
processing.LeaseExpiresAt = DateTime.UtcNow.AddMinutes(-1);
db.AddRange(pending, processing);
await db.SaveChangesAsync();
var snapshot = await new BackgroundJobMonitoringService(db)
.GetSnapshotAsync(CancellationToken.None);
Assert.Equal(1, snapshot.Pending);
Assert.Equal(1, snapshot.Processing);
Assert.Equal(1, snapshot.ExpiredLeases);
Assert.NotNull(snapshot.OldestUnfinishedAt);
Assert.True(snapshot.OldestUnfinishedAgeSeconds >= 100);
}
finally
{
File.Delete(databasePath);
}
}
[Fact] [Fact]
public async Task Runner_stops_after_retry_limit_and_ignores_duplicate_delivery() public async Task Runner_stops_after_retry_limit_and_ignores_duplicate_delivery()
{ {
@@ -109,6 +191,7 @@ public sealed class BackgroundJobOutboxTests
services.AddSingleton<InMemoryBackgroundJobTransport>(); services.AddSingleton<InMemoryBackgroundJobTransport>();
services.AddSingleton<IBackgroundJobTransport>(provider => services.AddSingleton<IBackgroundJobTransport>(provider =>
provider.GetRequiredService<InMemoryBackgroundJobTransport>()); provider.GetRequiredService<InMemoryBackgroundJobTransport>());
services.AddSingleton<BackgroundJobTelemetry>();
services.AddSingleton<BackgroundJobRunner>(); services.AddSingleton<BackgroundJobRunner>();
var provider = services.BuildServiceProvider(); var provider = services.BuildServiceProvider();