第二阶段优化已完成,重点是提高后台任务吞吐并降低 Outbox 数据库开销。
RabbitMQ 默认从单一回调通道提升为三类任务独立并行,跨类型并发能力由 1 提升到 3;每类还能独立配置 1–16 个消费者。[RabbitMqBackgroundJobs.cs (line 143)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/RabbitMqBackgroundJobs.cs:143) InMemory 开发模式同步改为按任务类型隔离队列,避免某类长任务堵塞其他任务。 租约恢复检查从“每发布一条执行一次”改为默认每 60 秒维护一次。 启动恢复改为数据库 NOT EXISTS 查询,不再把全部历史 Outbox 加载进内存。[BackgroundJobOutboxPublisher.cs (line 9)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/BackgroundJobOutboxPublisher.cs:9) 完成消息默认保留 14 天,之后按每批 500 条清理,并增加对应组合索引。 /health/messaging 现在返回各状态积压量、过期租约和最老任务等待时间。[BackgroundJobMonitoringService.cs (line 16)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/BackgroundJobMonitoringService.cs:16) 新增 Jiaowu.BackgroundJobs 运行时指标,覆盖发布量、处理量、发布耗时、处理耗时和清理量。[BackgroundJobTelemetry.cs (line 9)](E:/jiaowu/src/Jiaowu.Api/Infrastructure/BackgroundJobs/BackgroundJobTelemetry.cs:9) 配置、Compose 和调优建议已更新。[README.md (line 257)](E:/jiaowu/README.md:257)
This commit is contained in:
@@ -10,6 +10,9 @@ MYSQL_ROOT_PASSWORD=
|
|||||||
|
|
||||||
RABBITMQ_USER=jiaowu
|
RABBITMQ_USER=jiaowu
|
||||||
RABBITMQ_PASSWORD=
|
RABBITMQ_PASSWORD=
|
||||||
|
BACKGROUND_JOB_AUTOMATIC_SCHEDULE_CONCURRENCY=1
|
||||||
|
BACKGROUND_JOB_SCHEDULE_PUBLISH_CONCURRENCY=1
|
||||||
|
BACKGROUND_JOB_MAKEUP_EXAM_AUTO_CONCURRENCY=1
|
||||||
|
|
||||||
JWT_KEY=
|
JWT_KEY=
|
||||||
ALLOWED_HOSTS=localhost
|
ALLOWED_HOSTS=localhost
|
||||||
|
|||||||
@@ -11,6 +11,9 @@ ConnectionStrings__MySql="Server=db.example.edu.cn;Port=3306;Database=jiaowu;Use
|
|||||||
|
|
||||||
# 单机可继续使用 InMemory;多实例生产部署建议启用 RabbitMQ。
|
# 单机可继续使用 InMemory;多实例生产部署建议启用 RabbitMQ。
|
||||||
BackgroundJobs__Transport=InMemory
|
BackgroundJobs__Transport=InMemory
|
||||||
|
BackgroundJobs__AutomaticScheduleConcurrency=1
|
||||||
|
BackgroundJobs__SchedulePublishConcurrency=1
|
||||||
|
BackgroundJobs__MakeupExamAutoConcurrency=1
|
||||||
# RabbitMq__HostName=rabbitmq.example.edu.cn
|
# RabbitMq__HostName=rabbitmq.example.edu.cn
|
||||||
# RabbitMq__Port=5671
|
# RabbitMq__Port=5671
|
||||||
# RabbitMq__UserName=jiaowu
|
# RabbitMq__UserName=jiaowu
|
||||||
|
|||||||
@@ -224,7 +224,8 @@ SQLite 只用于本地开发:新库通过 `EnsureCreated` 建立,已有开
|
|||||||
- `/health/cache`:检查可选 Redis;未配置 Redis 时返回 `disabled`,Redis
|
- `/health/cache`:检查可选 Redis;未配置 Redis 时返回 `disabled`,Redis
|
||||||
故障不会影响数据库就绪探针。
|
故障不会影响数据库就绪探针。
|
||||||
- `/health/messaging`:检查后台任务传输;单机内存队列返回 `memory`,启用
|
- `/health/messaging`:检查后台任务传输;单机内存队列返回 `memory`,启用
|
||||||
RabbitMQ 时实际检查代理连接。
|
RabbitMQ 时实际检查代理连接,同时返回各 Outbox 状态的积压数、过期租约数和
|
||||||
|
最老未完成任务的等待时间。
|
||||||
|
|
||||||
### 查询缓存与 Redis
|
### 查询缓存与 Redis
|
||||||
|
|
||||||
@@ -270,6 +271,9 @@ BackgroundJobs__Transport=InMemory
|
|||||||
|
|
||||||
```text
|
```text
|
||||||
BackgroundJobs__Transport=RabbitMq
|
BackgroundJobs__Transport=RabbitMq
|
||||||
|
BackgroundJobs__AutomaticScheduleConcurrency=1
|
||||||
|
BackgroundJobs__SchedulePublishConcurrency=1
|
||||||
|
BackgroundJobs__MakeupExamAutoConcurrency=1
|
||||||
RabbitMq__HostName=rabbitmq.example.edu.cn
|
RabbitMq__HostName=rabbitmq.example.edu.cn
|
||||||
RabbitMq__Port=5671
|
RabbitMq__Port=5671
|
||||||
RabbitMq__UserName=jiaowu
|
RabbitMq__UserName=jiaowu
|
||||||
@@ -280,9 +284,18 @@ RabbitMq__TlsServerName=rabbitmq.example.edu.cn
|
|||||||
```
|
```
|
||||||
|
|
||||||
RabbitMQ 传输使用持久消息、发布确认、手动消费确认、每种任务独立队列和死信队列。
|
RabbitMQ 传输使用持久消息、发布确认、手动消费确认、每种任务独立队列和死信队列。
|
||||||
默认创建 Quorum Queue,重任务的消费者预取数为 1。MySQL 或 RabbitMQ 暂时不可用时,
|
默认创建 Quorum Queue,重任务的消费者预取数为 1。三类任务各自至少有一个消费者,
|
||||||
未完成消息会根据 Outbox 状态和租约继续补投,消费者必须保持业务处理幂等。迁移服务
|
因此不同类型的任务不会再互相阻塞;单类任务的并发度可独立设置为 1-16。自动排课
|
||||||
应先应用 `BackgroundJobOutbox` 数据库迁移,再启动应用实例。
|
通常最消耗 CPU,建议先保持为 1,再根据 CPU、数据库连接池和任务等待时间逐级调到
|
||||||
|
2 或 3;不要只提高预取数。
|
||||||
|
|
||||||
|
Outbox 租约恢复改为按维护周期执行,避免积压发布时每条消息都额外扫描数据库;已完成
|
||||||
|
消息默认保留 14 天并按每批 500 条清理,可使用 `CompletedRetentionDays`、
|
||||||
|
`MaintenanceIntervalSeconds` 和 `CleanupBatchSize` 调整。应用暴露
|
||||||
|
`Jiaowu.BackgroundJobs` Meter,其中包含发布量、处理量、发布耗时、处理耗时和清理量,
|
||||||
|
可接入现有 OpenTelemetry/运行时指标采集器。MySQL 或 RabbitMQ 暂时不可用时,未完成
|
||||||
|
消息会根据 Outbox 状态和租约继续补投。迁移服务应先应用
|
||||||
|
`BackgroundJobOutbox` 数据库迁移,再启动应用实例。
|
||||||
|
|
||||||
## 跨平台发布与 Docker
|
## 跨平台发布与 Docker
|
||||||
|
|
||||||
|
|||||||
@@ -17,6 +17,9 @@ x-jiaowu-environment: &jiaowu-environment
|
|||||||
Cache__Enabled: "true"
|
Cache__Enabled: "true"
|
||||||
Cache__KeyPrefix: "jiaowu:v1"
|
Cache__KeyPrefix: "jiaowu:v1"
|
||||||
BackgroundJobs__Transport: RabbitMq
|
BackgroundJobs__Transport: RabbitMq
|
||||||
|
BackgroundJobs__AutomaticScheduleConcurrency: "${BACKGROUND_JOB_AUTOMATIC_SCHEDULE_CONCURRENCY:-1}"
|
||||||
|
BackgroundJobs__SchedulePublishConcurrency: "${BACKGROUND_JOB_SCHEDULE_PUBLISH_CONCURRENCY:-1}"
|
||||||
|
BackgroundJobs__MakeupExamAutoConcurrency: "${BACKGROUND_JOB_MAKEUP_EXAM_AUTO_CONCURRENCY:-1}"
|
||||||
RabbitMq__HostName: rabbitmq
|
RabbitMq__HostName: rabbitmq
|
||||||
RabbitMq__Port: "5672"
|
RabbitMq__Port: "5672"
|
||||||
RabbitMq__UserName: "${RABBITMQ_USER:-jiaowu}"
|
RabbitMq__UserName: "${RABBITMQ_USER:-jiaowu}"
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
using Jiaowu.Api.Domain.System;
|
||||||
|
using Jiaowu.Api.Infrastructure.Persistence;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
|
||||||
|
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||||
|
|
||||||
|
public sealed record BackgroundJobBacklogSnapshot(
|
||||||
|
int Pending,
|
||||||
|
int Publishing,
|
||||||
|
int Published,
|
||||||
|
int Processing,
|
||||||
|
int ExpiredLeases,
|
||||||
|
DateTime? OldestUnfinishedAt,
|
||||||
|
double? OldestUnfinishedAgeSeconds);
|
||||||
|
|
||||||
|
public sealed class BackgroundJobMonitoringService(AppDbContext db)
|
||||||
|
{
|
||||||
|
public async Task<BackgroundJobBacklogSnapshot> GetSnapshotAsync(
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var counts = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||||
|
.Where(x => x.State != BackgroundJobOutboxState.Completed)
|
||||||
|
.GroupBy(x => x.State)
|
||||||
|
.Select(group => new { State = group.Key, Count = group.Count() })
|
||||||
|
.ToListAsync(cancellationToken);
|
||||||
|
var countByState = counts.ToDictionary(x => x.State, x => x.Count);
|
||||||
|
var now = DateTime.UtcNow;
|
||||||
|
var expiredLeases = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||||
|
.CountAsync(
|
||||||
|
x => (x.State == BackgroundJobOutboxState.Publishing ||
|
||||||
|
x.State == BackgroundJobOutboxState.Processing) &&
|
||||||
|
x.LeaseExpiresAt != null &&
|
||||||
|
x.LeaseExpiresAt < now,
|
||||||
|
cancellationToken);
|
||||||
|
var oldestUnfinishedAt = await db.BackgroundJobOutboxMessages
|
||||||
|
.AsNoTracking()
|
||||||
|
.Where(x => x.State != BackgroundJobOutboxState.Completed)
|
||||||
|
.Select(x => (DateTime?)x.CreatedAt)
|
||||||
|
.MinAsync(cancellationToken);
|
||||||
|
double? ageSeconds = oldestUnfinishedAt.HasValue
|
||||||
|
? Math.Max(0, (now - oldestUnfinishedAt.Value).TotalSeconds)
|
||||||
|
: null;
|
||||||
|
|
||||||
|
return new BackgroundJobBacklogSnapshot(
|
||||||
|
GetCount(BackgroundJobOutboxState.Pending),
|
||||||
|
GetCount(BackgroundJobOutboxState.Publishing),
|
||||||
|
GetCount(BackgroundJobOutboxState.Published),
|
||||||
|
GetCount(BackgroundJobOutboxState.Processing),
|
||||||
|
expiredLeases,
|
||||||
|
oldestUnfinishedAt,
|
||||||
|
ageSeconds);
|
||||||
|
|
||||||
|
int GetCount(BackgroundJobOutboxState state) =>
|
||||||
|
countByState.GetValueOrDefault(state);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,3 +1,5 @@
|
|||||||
|
using Jiaowu.Api.Domain.System;
|
||||||
|
|
||||||
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||||
|
|
||||||
public sealed class BackgroundJobOptions
|
public sealed class BackgroundJobOptions
|
||||||
@@ -8,13 +10,27 @@ public sealed class BackgroundJobOptions
|
|||||||
public int PollIntervalMilliseconds { get; set; } = 500;
|
public int PollIntervalMilliseconds { get; set; } = 500;
|
||||||
public int LeaseSeconds { get; set; } = 120;
|
public int LeaseSeconds { get; set; } = 120;
|
||||||
public ushort PrefetchCount { get; set; } = 1;
|
public ushort PrefetchCount { get; set; } = 1;
|
||||||
|
public int AutomaticScheduleConcurrency { get; set; } = 1;
|
||||||
|
public int SchedulePublishConcurrency { get; set; } = 1;
|
||||||
|
public int MakeupExamAutoConcurrency { get; set; } = 1;
|
||||||
public string Exchange { get; set; } = "jiaowu.background-jobs";
|
public string Exchange { get; set; } = "jiaowu.background-jobs";
|
||||||
public string QueuePrefix { get; set; } = "jiaowu.background-jobs";
|
public string QueuePrefix { get; set; } = "jiaowu.background-jobs";
|
||||||
public bool UseQuorumQueues { get; set; } = true;
|
public bool UseQuorumQueues { get; set; } = true;
|
||||||
public int ProcessingAttemptLimit { get; set; } = 5;
|
public int ProcessingAttemptLimit { get; set; } = 5;
|
||||||
|
public int MaintenanceIntervalSeconds { get; set; } = 60;
|
||||||
|
public int CompletedRetentionDays { get; set; } = 14;
|
||||||
|
public int CleanupBatchSize { get; set; } = 500;
|
||||||
|
|
||||||
public bool UsesRabbitMq =>
|
public bool UsesRabbitMq =>
|
||||||
Transport.Equals("RabbitMq", StringComparison.OrdinalIgnoreCase);
|
Transport.Equals("RabbitMq", StringComparison.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
public int ConsumerConcurrency(BackgroundJobKind kind) => kind switch
|
||||||
|
{
|
||||||
|
BackgroundJobKind.AutomaticSchedule => AutomaticScheduleConcurrency,
|
||||||
|
BackgroundJobKind.SchedulePublish => SchedulePublishConcurrency,
|
||||||
|
BackgroundJobKind.MakeupExamAuto => MakeupExamAutoConcurrency,
|
||||||
|
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null)
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
public sealed class RabbitMqOptions
|
public sealed class RabbitMqOptions
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
using System.Diagnostics;
|
||||||
using Jiaowu.Api.Domain.Academic;
|
using Jiaowu.Api.Domain.Academic;
|
||||||
using Jiaowu.Api.Domain.System;
|
using Jiaowu.Api.Domain.System;
|
||||||
using Jiaowu.Api.Infrastructure.Persistence;
|
using Jiaowu.Api.Infrastructure.Persistence;
|
||||||
@@ -9,17 +10,26 @@ public sealed class BackgroundJobOutboxPublisher(
|
|||||||
IServiceScopeFactory scopeFactory,
|
IServiceScopeFactory scopeFactory,
|
||||||
IBackgroundJobTransport transport,
|
IBackgroundJobTransport transport,
|
||||||
BackgroundJobOptions options,
|
BackgroundJobOptions options,
|
||||||
|
BackgroundJobTelemetry telemetry,
|
||||||
ILogger<BackgroundJobOutboxPublisher> logger) : BackgroundService
|
ILogger<BackgroundJobOutboxPublisher> logger) : BackgroundService
|
||||||
{
|
{
|
||||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
{
|
{
|
||||||
await PrepareForStartupAsync(stoppingToken);
|
await PrepareForStartupAsync(stoppingToken);
|
||||||
|
var nextMaintenanceAt = DateTime.MinValue;
|
||||||
|
|
||||||
while (!stoppingToken.IsCancellationRequested)
|
while (!stoppingToken.IsCancellationRequested)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
|
{
|
||||||
|
if (DateTime.UtcNow >= nextMaintenanceAt)
|
||||||
{
|
{
|
||||||
await RecoverExpiredProcessingAsync(stoppingToken);
|
await RecoverExpiredProcessingAsync(stoppingToken);
|
||||||
|
await CleanupCompletedAsync(stoppingToken);
|
||||||
|
nextMaintenanceAt = DateTime.UtcNow.AddSeconds(
|
||||||
|
options.MaintenanceIntervalSeconds);
|
||||||
|
}
|
||||||
|
|
||||||
var claimed = await ClaimNextAsync(stoppingToken);
|
var claimed = await ClaimNextAsync(stoppingToken);
|
||||||
if (claimed is null)
|
if (claimed is null)
|
||||||
{
|
{
|
||||||
@@ -62,51 +72,107 @@ public sealed class BackgroundJobOutboxPublisher(
|
|||||||
await AddMissingOutboxMessagesAsync(db, cancellationToken);
|
await AddMissingOutboxMessagesAsync(db, cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static async Task AddMissingOutboxMessagesAsync(
|
private async Task AddMissingOutboxMessagesAsync(
|
||||||
AppDbContext db,
|
AppDbContext db,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
var existing = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
|
||||||
.Select(x => new { x.JobKind, x.JobId })
|
|
||||||
.ToListAsync(cancellationToken);
|
|
||||||
var existingKeys = existing
|
|
||||||
.Select(x => (x.JobKind, x.JobId))
|
|
||||||
.ToHashSet();
|
|
||||||
|
|
||||||
var automaticJobs = await db.AutomaticScheduleJobs.AsNoTracking()
|
var automaticJobs = await db.AutomaticScheduleJobs.AsNoTracking()
|
||||||
.Where(x => x.Status == AutomaticScheduleJobStatus.Queued ||
|
.Where(x =>
|
||||||
x.Status == AutomaticScheduleJobStatus.Running)
|
(x.Status == AutomaticScheduleJobStatus.Queued ||
|
||||||
|
x.Status == AutomaticScheduleJobStatus.Running) &&
|
||||||
|
!db.BackgroundJobOutboxMessages.Any(message =>
|
||||||
|
message.JobKind == BackgroundJobKind.AutomaticSchedule &&
|
||||||
|
message.JobId == x.Id))
|
||||||
.Select(x => x.Id)
|
.Select(x => x.Id)
|
||||||
.ToListAsync(cancellationToken);
|
.ToListAsync(cancellationToken);
|
||||||
var publishJobs = await db.SchedulePublishJobs.AsNoTracking()
|
var publishJobs = await db.SchedulePublishJobs.AsNoTracking()
|
||||||
.Where(x => x.Status == SchedulePublishJobStatus.Queued ||
|
.Where(x =>
|
||||||
x.Status == SchedulePublishJobStatus.Running)
|
(x.Status == SchedulePublishJobStatus.Queued ||
|
||||||
|
x.Status == SchedulePublishJobStatus.Running) &&
|
||||||
|
!db.BackgroundJobOutboxMessages.Any(message =>
|
||||||
|
message.JobKind == BackgroundJobKind.SchedulePublish &&
|
||||||
|
message.JobId == x.Id))
|
||||||
.Select(x => x.Id)
|
.Select(x => x.Id)
|
||||||
.ToListAsync(cancellationToken);
|
.ToListAsync(cancellationToken);
|
||||||
var makeupJobs = await db.MakeupExamAutoJobs.AsNoTracking()
|
var makeupJobs = await db.MakeupExamAutoJobs.AsNoTracking()
|
||||||
.Where(x => x.Status == MakeupExamAutoJobStatus.Queued ||
|
.Where(x =>
|
||||||
x.Status == MakeupExamAutoJobStatus.Running)
|
(x.Status == MakeupExamAutoJobStatus.Queued ||
|
||||||
|
x.Status == MakeupExamAutoJobStatus.Running) &&
|
||||||
|
!db.BackgroundJobOutboxMessages.Any(message =>
|
||||||
|
message.JobKind == BackgroundJobKind.MakeupExamAuto &&
|
||||||
|
message.JobId == x.Id))
|
||||||
.Select(x => x.Id)
|
.Select(x => x.Id)
|
||||||
.ToListAsync(cancellationToken);
|
.ToListAsync(cancellationToken);
|
||||||
|
|
||||||
AddMissing(BackgroundJobKind.AutomaticSchedule, automaticJobs);
|
var missingKeys = automaticJobs
|
||||||
AddMissing(BackgroundJobKind.SchedulePublish, publishJobs);
|
.Select(id => (BackgroundJobKind.AutomaticSchedule, id))
|
||||||
AddMissing(BackgroundJobKind.MakeupExamAuto, makeupJobs);
|
.Concat(publishJobs.Select(id =>
|
||||||
|
(BackgroundJobKind.SchedulePublish, id)))
|
||||||
if (db.ChangeTracker.HasChanges())
|
.Concat(makeupJobs.Select(id =>
|
||||||
await db.SaveChangesAsync(cancellationToken);
|
(BackgroundJobKind.MakeupExamAuto, id)))
|
||||||
|
.ToList();
|
||||||
void AddMissing(BackgroundJobKind kind, IEnumerable<Guid> jobIds)
|
foreach (var (kind, jobId) in missingKeys)
|
||||||
{
|
|
||||||
foreach (var jobId in jobIds)
|
|
||||||
{
|
|
||||||
if (existingKeys.Add((kind, jobId)))
|
|
||||||
{
|
{
|
||||||
db.BackgroundJobOutboxMessages.Add(
|
db.BackgroundJobOutboxMessages.Add(
|
||||||
BackgroundJobOutboxMessage.Create(kind, jobId));
|
BackgroundJobOutboxMessage.Create(kind, jobId));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!db.ChangeTracker.HasChanges())
|
||||||
|
return;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await db.SaveChangesAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
catch (DbUpdateException)
|
||||||
|
{
|
||||||
|
db.ChangeTracker.Clear();
|
||||||
|
foreach (var (kind, jobId) in missingKeys)
|
||||||
|
{
|
||||||
|
var recovered = await db.BackgroundJobOutboxMessages
|
||||||
|
.AsNoTracking()
|
||||||
|
.AnyAsync(
|
||||||
|
message => message.JobKind == kind &&
|
||||||
|
message.JobId == jobId,
|
||||||
|
cancellationToken);
|
||||||
|
if (!recovered)
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.LogInformation(
|
||||||
|
"Another application instance recovered {Count} missing outbox messages first.",
|
||||||
|
missingKeys.Count);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async Task CleanupCompletedAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await using var scope = scopeFactory.CreateAsyncScope();
|
||||||
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||||
|
var cutoff = DateTime.UtcNow.AddDays(-options.CompletedRetentionDays);
|
||||||
|
var completedIds = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||||
|
.Where(x =>
|
||||||
|
x.State == BackgroundJobOutboxState.Completed &&
|
||||||
|
x.CompletedAt != null &&
|
||||||
|
x.CompletedAt < cutoff)
|
||||||
|
.OrderBy(x => x.CompletedAt)
|
||||||
|
.Select(x => x.Id)
|
||||||
|
.Take(options.CleanupBatchSize)
|
||||||
|
.ToListAsync(cancellationToken);
|
||||||
|
if (completedIds.Count == 0)
|
||||||
|
return;
|
||||||
|
|
||||||
|
foreach (var id in completedIds)
|
||||||
|
{
|
||||||
|
db.BackgroundJobOutboxMessages.Remove(
|
||||||
|
new BackgroundJobOutboxMessage { Id = id });
|
||||||
|
}
|
||||||
|
await db.SaveChangesAsync(cancellationToken);
|
||||||
|
telemetry.RecordCleaned(completedIds.Count);
|
||||||
|
logger.LogInformation(
|
||||||
|
"Removed {Count} completed background job outbox messages older than {Cutoff}.",
|
||||||
|
completedIds.Count,
|
||||||
|
cutoff);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task RecoverExpiredProcessingAsync(
|
private async Task RecoverExpiredProcessingAsync(
|
||||||
@@ -175,6 +241,7 @@ public sealed class BackgroundJobOutboxPublisher(
|
|||||||
Guid token,
|
Guid token,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
|
var startedAt = Stopwatch.GetTimestamp();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
await transport.PublishAsync(message, cancellationToken);
|
await transport.PublishAsync(message, cancellationToken);
|
||||||
@@ -193,6 +260,11 @@ public sealed class BackgroundJobOutboxPublisher(
|
|||||||
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||||
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null),
|
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null),
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
|
telemetry.RecordPublish(
|
||||||
|
message.JobKind,
|
||||||
|
transport.IsDurable,
|
||||||
|
"published",
|
||||||
|
Stopwatch.GetElapsedTime(startedAt));
|
||||||
}
|
}
|
||||||
catch (Exception exception)
|
catch (Exception exception)
|
||||||
{
|
{
|
||||||
@@ -213,6 +285,11 @@ public sealed class BackgroundJobOutboxPublisher(
|
|||||||
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null)
|
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null)
|
||||||
.SetProperty(x => x.LastError, messageText),
|
.SetProperty(x => x.LastError, messageText),
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
|
telemetry.RecordPublish(
|
||||||
|
message.JobKind,
|
||||||
|
transport.IsDurable,
|
||||||
|
"failed",
|
||||||
|
Stopwatch.GetElapsedTime(startedAt));
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
using System.Diagnostics;
|
||||||
using Jiaowu.Api.Domain.Academic;
|
using Jiaowu.Api.Domain.Academic;
|
||||||
using Jiaowu.Api.Domain.System;
|
using Jiaowu.Api.Domain.System;
|
||||||
using Jiaowu.Api.Infrastructure.Exams;
|
using Jiaowu.Api.Infrastructure.Exams;
|
||||||
@@ -28,12 +29,14 @@ public sealed class BackgroundJobRunner(
|
|||||||
IServiceScopeFactory scopeFactory,
|
IServiceScopeFactory scopeFactory,
|
||||||
IBackgroundJobTransport transport,
|
IBackgroundJobTransport transport,
|
||||||
BackgroundJobOptions options,
|
BackgroundJobOptions options,
|
||||||
|
BackgroundJobTelemetry telemetry,
|
||||||
ILogger<BackgroundJobRunner> logger)
|
ILogger<BackgroundJobRunner> logger)
|
||||||
{
|
{
|
||||||
public async Task<BackgroundJobRunResult> RunAsync(
|
public async Task<BackgroundJobRunResult> RunAsync(
|
||||||
BackgroundJobEnvelope message,
|
BackgroundJobEnvelope message,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
|
var startedAt = Stopwatch.GetTimestamp();
|
||||||
var token = Guid.NewGuid();
|
var token = Guid.NewGuid();
|
||||||
var claim = await TryClaimAsync(message, token, cancellationToken);
|
var claim = await TryClaimAsync(message, token, cancellationToken);
|
||||||
if (!claim.Claimed)
|
if (!claim.Claimed)
|
||||||
@@ -48,6 +51,10 @@ public sealed class BackgroundJobRunner(
|
|||||||
await MarkJobRetryLimitExceededAsync(message, cancellationToken);
|
await MarkJobRetryLimitExceededAsync(message, cancellationToken);
|
||||||
await MarkCompletedAsync(message.OutboxMessageId, token,
|
await MarkCompletedAsync(message.OutboxMessageId, token,
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
|
telemetry.RecordProcessing(
|
||||||
|
message.JobKind,
|
||||||
|
"retry-limit",
|
||||||
|
Stopwatch.GetElapsedTime(startedAt));
|
||||||
return BackgroundJobRunResult.Completed;
|
return BackgroundJobRunResult.Completed;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -85,6 +92,10 @@ public sealed class BackgroundJobRunner(
|
|||||||
|
|
||||||
await MarkCompletedAsync(message.OutboxMessageId, token,
|
await MarkCompletedAsync(message.OutboxMessageId, token,
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
|
telemetry.RecordProcessing(
|
||||||
|
message.JobKind,
|
||||||
|
"completed",
|
||||||
|
Stopwatch.GetElapsedTime(startedAt));
|
||||||
return BackgroundJobRunResult.Completed;
|
return BackgroundJobRunResult.Completed;
|
||||||
}
|
}
|
||||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||||
@@ -103,6 +114,10 @@ public sealed class BackgroundJobRunner(
|
|||||||
token,
|
token,
|
||||||
exception,
|
exception,
|
||||||
CancellationToken.None);
|
CancellationToken.None);
|
||||||
|
telemetry.RecordProcessing(
|
||||||
|
message.JobKind,
|
||||||
|
"retry",
|
||||||
|
Stopwatch.GetElapsedTime(startedAt));
|
||||||
return BackgroundJobRunResult.Retry(TimeSpan.FromSeconds(2));
|
return BackgroundJobRunResult.Retry(TimeSpan.FromSeconds(2));
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
using System.Diagnostics;
|
||||||
|
using System.Diagnostics.Metrics;
|
||||||
|
using Jiaowu.Api.Domain.System;
|
||||||
|
|
||||||
|
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||||
|
|
||||||
|
public sealed class BackgroundJobTelemetry : IDisposable
|
||||||
|
{
|
||||||
|
public const string MeterName = "Jiaowu.BackgroundJobs";
|
||||||
|
|
||||||
|
private readonly Meter _meter = new(MeterName, "1.0.0");
|
||||||
|
private readonly Counter<long> _published;
|
||||||
|
private readonly Counter<long> _processed;
|
||||||
|
private readonly Histogram<double> _publishDuration;
|
||||||
|
private readonly Histogram<double> _processingDuration;
|
||||||
|
private readonly Counter<long> _cleaned;
|
||||||
|
|
||||||
|
public BackgroundJobTelemetry()
|
||||||
|
{
|
||||||
|
_published = _meter.CreateCounter<long>(
|
||||||
|
"jiaowu.background_jobs.published",
|
||||||
|
unit: "{message}");
|
||||||
|
_processed = _meter.CreateCounter<long>(
|
||||||
|
"jiaowu.background_jobs.processed",
|
||||||
|
unit: "{job}");
|
||||||
|
_publishDuration = _meter.CreateHistogram<double>(
|
||||||
|
"jiaowu.background_jobs.publish.duration",
|
||||||
|
unit: "ms");
|
||||||
|
_processingDuration = _meter.CreateHistogram<double>(
|
||||||
|
"jiaowu.background_jobs.processing.duration",
|
||||||
|
unit: "ms");
|
||||||
|
_cleaned = _meter.CreateCounter<long>(
|
||||||
|
"jiaowu.background_jobs.outbox.cleaned",
|
||||||
|
unit: "{message}");
|
||||||
|
}
|
||||||
|
|
||||||
|
public void RecordPublish(
|
||||||
|
BackgroundJobKind kind,
|
||||||
|
bool durable,
|
||||||
|
string outcome,
|
||||||
|
TimeSpan duration)
|
||||||
|
{
|
||||||
|
var tags = new TagList
|
||||||
|
{
|
||||||
|
{ "job.kind", kind.ToString() },
|
||||||
|
{ "messaging.backend", durable ? "rabbitmq" : "memory" },
|
||||||
|
{ "job.outcome", outcome }
|
||||||
|
};
|
||||||
|
_published.Add(1, tags);
|
||||||
|
_publishDuration.Record(duration.TotalMilliseconds, tags);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void RecordProcessing(
|
||||||
|
BackgroundJobKind kind,
|
||||||
|
string outcome,
|
||||||
|
TimeSpan duration)
|
||||||
|
{
|
||||||
|
var tags = new TagList
|
||||||
|
{
|
||||||
|
{ "job.kind", kind.ToString() },
|
||||||
|
{ "job.outcome", outcome }
|
||||||
|
};
|
||||||
|
_processed.Add(1, tags);
|
||||||
|
_processingDuration.Record(duration.TotalMilliseconds, tags);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void RecordCleaned(int count)
|
||||||
|
{
|
||||||
|
if (count > 0)
|
||||||
|
_cleaned.Add(count);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose() => _meter.Dispose();
|
||||||
|
}
|
||||||
@@ -21,26 +21,31 @@ public interface IBackgroundJobTransport
|
|||||||
|
|
||||||
public sealed class InMemoryBackgroundJobTransport : IBackgroundJobTransport
|
public sealed class InMemoryBackgroundJobTransport : IBackgroundJobTransport
|
||||||
{
|
{
|
||||||
private readonly Channel<BackgroundJobEnvelope> _channel =
|
private readonly IReadOnlyDictionary<
|
||||||
Channel.CreateBounded<BackgroundJobEnvelope>(
|
BackgroundJobKind,
|
||||||
|
Channel<BackgroundJobEnvelope>> _channels =
|
||||||
|
Enum.GetValues<BackgroundJobKind>().ToDictionary(
|
||||||
|
kind => kind,
|
||||||
|
_ => Channel.CreateBounded<BackgroundJobEnvelope>(
|
||||||
new BoundedChannelOptions(256)
|
new BoundedChannelOptions(256)
|
||||||
{
|
{
|
||||||
FullMode = BoundedChannelFullMode.Wait,
|
FullMode = BoundedChannelFullMode.Wait,
|
||||||
SingleReader = true,
|
SingleReader = false,
|
||||||
SingleWriter = false
|
SingleWriter = false
|
||||||
});
|
}));
|
||||||
|
|
||||||
public bool IsDurable => false;
|
public bool IsDurable => false;
|
||||||
|
|
||||||
public ValueTask PublishAsync(
|
public ValueTask PublishAsync(
|
||||||
BackgroundJobEnvelope message,
|
BackgroundJobEnvelope message,
|
||||||
CancellationToken cancellationToken) =>
|
CancellationToken cancellationToken) =>
|
||||||
_channel.Writer.WriteAsync(message, cancellationToken);
|
_channels[message.JobKind].Writer.WriteAsync(message, cancellationToken);
|
||||||
|
|
||||||
public Task<bool> CheckHealthAsync(CancellationToken cancellationToken) =>
|
public Task<bool> CheckHealthAsync(CancellationToken cancellationToken) =>
|
||||||
Task.FromResult(true);
|
Task.FromResult(true);
|
||||||
|
|
||||||
public IAsyncEnumerable<BackgroundJobEnvelope> ReadAllAsync(
|
public IAsyncEnumerable<BackgroundJobEnvelope> ReadAllAsync(
|
||||||
|
BackgroundJobKind kind,
|
||||||
CancellationToken cancellationToken) =>
|
CancellationToken cancellationToken) =>
|
||||||
_channel.Reader.ReadAllAsync(cancellationToken);
|
_channels[kind].Reader.ReadAllAsync(cancellationToken);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,27 +1,46 @@
|
|||||||
|
using Jiaowu.Api.Domain.System;
|
||||||
|
|
||||||
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||||
|
|
||||||
public sealed class InMemoryBackgroundJobWorker(
|
public sealed class InMemoryBackgroundJobWorker(
|
||||||
InMemoryBackgroundJobTransport transport,
|
InMemoryBackgroundJobTransport transport,
|
||||||
BackgroundJobRunner runner,
|
BackgroundJobRunner runner,
|
||||||
|
BackgroundJobOptions options,
|
||||||
ILogger<InMemoryBackgroundJobWorker> logger) : BackgroundService
|
ILogger<InMemoryBackgroundJobWorker> logger) : BackgroundService
|
||||||
{
|
{
|
||||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
{
|
{
|
||||||
|
var workers = Enum.GetValues<BackgroundJobKind>()
|
||||||
|
.SelectMany(kind => Enumerable.Range(
|
||||||
|
0,
|
||||||
|
options.ConsumerConcurrency(kind))
|
||||||
|
.Select(_ => ConsumeAsync(kind, stoppingToken)))
|
||||||
|
.ToArray();
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
await foreach (var message in transport.ReadAllAsync(stoppingToken))
|
await Task.WhenAll(workers);
|
||||||
{
|
|
||||||
var result = await runner.RunAsync(message, stoppingToken);
|
|
||||||
if (result.Outcome != BackgroundJobRunOutcome.Retry)
|
|
||||||
continue;
|
|
||||||
|
|
||||||
await Task.Delay(result.RetryAfter, stoppingToken);
|
|
||||||
await transport.PublishAsync(message, stoppingToken);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||||
{
|
{
|
||||||
logger.LogInformation("In-memory background job worker is stopping.");
|
logger.LogInformation("In-memory background job worker is stopping.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async Task ConsumeAsync(
|
||||||
|
BackgroundJobKind kind,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await foreach (var message in transport.ReadAllAsync(
|
||||||
|
kind,
|
||||||
|
cancellationToken))
|
||||||
|
{
|
||||||
|
var result = await runner.RunAsync(message, cancellationToken);
|
||||||
|
if (result.Outcome != BackgroundJobRunOutcome.Retry)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
await Task.Delay(result.RetryAfter, cancellationToken);
|
||||||
|
await transport.PublishAsync(message, cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -82,6 +82,7 @@ public sealed class RabbitMqBackgroundJobTransport(
|
|||||||
_connection = await RabbitMqBackgroundJobTopology.CreateConnectionAsync(
|
_connection = await RabbitMqBackgroundJobTopology.CreateConnectionAsync(
|
||||||
rabbitOptions,
|
rabbitOptions,
|
||||||
"jiaowu-background-job-publisher",
|
"jiaowu-background-job-publisher",
|
||||||
|
consumerDispatchConcurrency: 1,
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
_channel = await _connection.CreateChannelAsync(
|
_channel = await _connection.CreateChannelAsync(
|
||||||
new CreateChannelOptions(
|
new CreateChannelOptions(
|
||||||
@@ -153,12 +154,19 @@ public sealed class RabbitMqBackgroundJobWorker(
|
|||||||
var channels = new List<IChannel>();
|
var channels = new List<IChannel>();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
|
var consumerCount = RabbitMqBackgroundJobTopology.JobKinds.Sum(
|
||||||
|
jobOptions.ConsumerConcurrency);
|
||||||
connection = await RabbitMqBackgroundJobTopology.CreateConnectionAsync(
|
connection = await RabbitMqBackgroundJobTopology.CreateConnectionAsync(
|
||||||
rabbitOptions,
|
rabbitOptions,
|
||||||
"jiaowu-background-job-worker",
|
"jiaowu-background-job-worker",
|
||||||
|
(ushort)consumerCount,
|
||||||
stoppingToken);
|
stoppingToken);
|
||||||
|
|
||||||
foreach (var kind in RabbitMqBackgroundJobTopology.JobKinds)
|
foreach (var kind in RabbitMqBackgroundJobTopology.JobKinds)
|
||||||
|
{
|
||||||
|
for (var workerIndex = 0;
|
||||||
|
workerIndex < jobOptions.ConsumerConcurrency(kind);
|
||||||
|
workerIndex++)
|
||||||
{
|
{
|
||||||
var channel = await connection.CreateChannelAsync(
|
var channel = await connection.CreateChannelAsync(
|
||||||
new CreateChannelOptions(
|
new CreateChannelOptions(
|
||||||
@@ -186,7 +194,8 @@ public sealed class RabbitMqBackgroundJobWorker(
|
|||||||
var deliveryToken = deliveryCancellation.Token;
|
var deliveryToken = deliveryCancellation.Token;
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var message = JsonSerializer.Deserialize<BackgroundJobEnvelope>(
|
var message = JsonSerializer
|
||||||
|
.Deserialize<BackgroundJobEnvelope>(
|
||||||
eventArgs.Body.Span);
|
eventArgs.Body.Span);
|
||||||
if (message is null || message.JobKind != kind)
|
if (message is null || message.JobKind != kind)
|
||||||
{
|
{
|
||||||
@@ -201,7 +210,8 @@ public sealed class RabbitMqBackgroundJobWorker(
|
|||||||
var result = await runner.RunAsync(
|
var result = await runner.RunAsync(
|
||||||
message,
|
message,
|
||||||
deliveryToken);
|
deliveryToken);
|
||||||
if (result.Outcome == BackgroundJobRunOutcome.Completed)
|
if (result.Outcome ==
|
||||||
|
BackgroundJobRunOutcome.Completed)
|
||||||
{
|
{
|
||||||
await channel.BasicAckAsync(
|
await channel.BasicAckAsync(
|
||||||
eventArgs.DeliveryTag,
|
eventArgs.DeliveryTag,
|
||||||
@@ -229,25 +239,29 @@ public sealed class RabbitMqBackgroundJobWorker(
|
|||||||
exception,
|
exception,
|
||||||
"RabbitMQ delivery for {JobKind} failed and will be requeued.",
|
"RabbitMQ delivery for {JobKind} failed and will be requeued.",
|
||||||
kind);
|
kind);
|
||||||
if (channel.IsOpen)
|
if (!channel.IsOpen)
|
||||||
{
|
return;
|
||||||
|
|
||||||
await channel.BasicNackAsync(
|
await channel.BasicNackAsync(
|
||||||
eventArgs.DeliveryTag,
|
eventArgs.DeliveryTag,
|
||||||
multiple: false,
|
multiple: false,
|
||||||
requeue: true,
|
requeue: true,
|
||||||
CancellationToken.None);
|
CancellationToken.None);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
};
|
};
|
||||||
await channel.BasicConsumeAsync(
|
await channel.BasicConsumeAsync(
|
||||||
RabbitMqBackgroundJobTopology.QueueName(jobOptions, kind),
|
RabbitMqBackgroundJobTopology.QueueName(
|
||||||
|
jobOptions,
|
||||||
|
kind),
|
||||||
autoAck: false,
|
autoAck: false,
|
||||||
consumer,
|
consumer,
|
||||||
stoppingToken);
|
stoppingToken);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
logger.LogInformation(
|
logger.LogInformation(
|
||||||
"RabbitMQ background job consumers are connected to {HostName}:{Port}.",
|
"{ConsumerCount} RabbitMQ background job consumers are connected to {HostName}:{Port}.",
|
||||||
|
consumerCount,
|
||||||
rabbitOptions.HostName,
|
rabbitOptions.HostName,
|
||||||
rabbitOptions.Port);
|
rabbitOptions.Port);
|
||||||
while (connection.IsOpen && !stoppingToken.IsCancellationRequested)
|
while (connection.IsOpen && !stoppingToken.IsCancellationRequested)
|
||||||
@@ -308,6 +322,7 @@ internal static class RabbitMqBackgroundJobTopology
|
|||||||
public static async Task<IConnection> CreateConnectionAsync(
|
public static async Task<IConnection> CreateConnectionAsync(
|
||||||
RabbitMqOptions options,
|
RabbitMqOptions options,
|
||||||
string clientName,
|
string clientName,
|
||||||
|
ushort consumerDispatchConcurrency,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
var factory = new ConnectionFactory
|
var factory = new ConnectionFactory
|
||||||
@@ -321,7 +336,7 @@ internal static class RabbitMqBackgroundJobTopology
|
|||||||
AutomaticRecoveryEnabled = false,
|
AutomaticRecoveryEnabled = false,
|
||||||
TopologyRecoveryEnabled = false,
|
TopologyRecoveryEnabled = false,
|
||||||
RequestedHeartbeat = TimeSpan.FromSeconds(30),
|
RequestedHeartbeat = TimeSpan.FromSeconds(30),
|
||||||
ConsumerDispatchConcurrency = 1
|
ConsumerDispatchConcurrency = consumerDispatchConcurrency
|
||||||
};
|
};
|
||||||
if (options.UseTls)
|
if (options.UseTls)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1000,6 +1000,7 @@ public sealed class AppDbContext(DbContextOptions<AppDbContext> options)
|
|||||||
entity.Property(x => x.LastError).HasMaxLength(2000);
|
entity.Property(x => x.LastError).HasMaxLength(2000);
|
||||||
entity.HasIndex(x => new { x.JobKind, x.JobId }).IsUnique();
|
entity.HasIndex(x => new { x.JobKind, x.JobId }).IsUnique();
|
||||||
entity.HasIndex(x => new { x.State, x.CreatedAt });
|
entity.HasIndex(x => new { x.State, x.CreatedAt });
|
||||||
|
entity.HasIndex(x => new { x.State, x.CompletedAt });
|
||||||
entity.HasIndex(x => x.LeaseExpiresAt);
|
entity.HasIndex(x => x.LeaseExpiresAt);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -2050,6 +2050,10 @@ public sealed class DevelopmentSqliteMigrator(
|
|||||||
"""
|
"""
|
||||||
CREATE INDEX "IX_BackgroundJobOutboxMessages_State_CreatedAt"
|
CREATE INDEX "IX_BackgroundJobOutboxMessages_State_CreatedAt"
|
||||||
ON "BackgroundJobOutboxMessages" ("State", "CreatedAt");
|
ON "BackgroundJobOutboxMessages" ("State", "CreatedAt");
|
||||||
|
""",
|
||||||
|
"""
|
||||||
|
CREATE INDEX "IX_BackgroundJobOutboxMessages_State_CompletedAt"
|
||||||
|
ON "BackgroundJobOutboxMessages" ("State", "CompletedAt");
|
||||||
"""
|
"""
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|||||||
+2
@@ -3478,6 +3478,8 @@ namespace Jiaowu.Api.Infrastructure.Persistence.Migrations.MySql
|
|||||||
|
|
||||||
b.HasIndex("State", "CreatedAt");
|
b.HasIndex("State", "CreatedAt");
|
||||||
|
|
||||||
|
b.HasIndex("State", "CompletedAt");
|
||||||
|
|
||||||
b.ToTable("BackgroundJobOutboxMessages");
|
b.ToTable("BackgroundJobOutboxMessages");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
+5
@@ -50,6 +50,11 @@ namespace Jiaowu.Api.Infrastructure.Persistence.Migrations.MySql
|
|||||||
name: "IX_BackgroundJobOutboxMessages_State_CreatedAt",
|
name: "IX_BackgroundJobOutboxMessages_State_CreatedAt",
|
||||||
table: "BackgroundJobOutboxMessages",
|
table: "BackgroundJobOutboxMessages",
|
||||||
columns: new[] { "State", "CreatedAt" });
|
columns: new[] { "State", "CreatedAt" });
|
||||||
|
|
||||||
|
migrationBuilder.CreateIndex(
|
||||||
|
name: "IX_BackgroundJobOutboxMessages_State_CompletedAt",
|
||||||
|
table: "BackgroundJobOutboxMessages",
|
||||||
|
columns: new[] { "State", "CompletedAt" });
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
|
|||||||
+2
@@ -3475,6 +3475,8 @@ namespace Jiaowu.Api.Infrastructure.Persistence.Migrations.MySql
|
|||||||
|
|
||||||
b.HasIndex("State", "CreatedAt");
|
b.HasIndex("State", "CreatedAt");
|
||||||
|
|
||||||
|
b.HasIndex("State", "CompletedAt");
|
||||||
|
|
||||||
b.ToTable("BackgroundJobOutboxMessages");
|
b.ToTable("BackgroundJobOutboxMessages");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -137,7 +137,13 @@ if (!backgroundJobOptions.Transport.Equals(
|
|||||||
if (backgroundJobOptions.PollIntervalMilliseconds is < 100 or > 30000 ||
|
if (backgroundJobOptions.PollIntervalMilliseconds is < 100 or > 30000 ||
|
||||||
backgroundJobOptions.LeaseSeconds is < 30 or > 3600 ||
|
backgroundJobOptions.LeaseSeconds is < 30 or > 3600 ||
|
||||||
backgroundJobOptions.PrefetchCount is < 1 or > 100 ||
|
backgroundJobOptions.PrefetchCount is < 1 or > 100 ||
|
||||||
|
backgroundJobOptions.AutomaticScheduleConcurrency is < 1 or > 16 ||
|
||||||
|
backgroundJobOptions.SchedulePublishConcurrency is < 1 or > 16 ||
|
||||||
|
backgroundJobOptions.MakeupExamAutoConcurrency is < 1 or > 16 ||
|
||||||
backgroundJobOptions.ProcessingAttemptLimit is < 1 or > 100 ||
|
backgroundJobOptions.ProcessingAttemptLimit is < 1 or > 100 ||
|
||||||
|
backgroundJobOptions.MaintenanceIntervalSeconds is < 10 or > 3600 ||
|
||||||
|
backgroundJobOptions.CompletedRetentionDays is < 1 or > 3650 ||
|
||||||
|
backgroundJobOptions.CleanupBatchSize is < 10 or > 5000 ||
|
||||||
string.IsNullOrWhiteSpace(backgroundJobOptions.Exchange) ||
|
string.IsNullOrWhiteSpace(backgroundJobOptions.Exchange) ||
|
||||||
string.IsNullOrWhiteSpace(backgroundJobOptions.QueuePrefix))
|
string.IsNullOrWhiteSpace(backgroundJobOptions.QueuePrefix))
|
||||||
{
|
{
|
||||||
@@ -269,6 +275,8 @@ builder.Services.AddScoped<ExamArrangementService>();
|
|||||||
builder.Services.AddScoped<MakeupExamEligibilityService>();
|
builder.Services.AddScoped<MakeupExamEligibilityService>();
|
||||||
builder.Services.AddScoped<MakeupExamArrangementService>();
|
builder.Services.AddScoped<MakeupExamArrangementService>();
|
||||||
builder.Services.AddScoped<MakeupExamAutoJobProcessor>();
|
builder.Services.AddScoped<MakeupExamAutoJobProcessor>();
|
||||||
|
builder.Services.AddSingleton<BackgroundJobTelemetry>();
|
||||||
|
builder.Services.AddScoped<BackgroundJobMonitoringService>();
|
||||||
builder.Services.AddSingleton<BackgroundJobRunner>();
|
builder.Services.AddSingleton<BackgroundJobRunner>();
|
||||||
if (backgroundJobOptions.UsesRabbitMq)
|
if (backgroundJobOptions.UsesRabbitMq)
|
||||||
{
|
{
|
||||||
@@ -534,15 +542,41 @@ static async Task<IResult> CheckCacheHealthAsync(
|
|||||||
|
|
||||||
static async Task<IResult> CheckMessagingHealthAsync(
|
static async Task<IResult> CheckMessagingHealthAsync(
|
||||||
IBackgroundJobTransport transport,
|
IBackgroundJobTransport transport,
|
||||||
|
BackgroundJobMonitoringService monitoring,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
var healthy = await transport.CheckHealthAsync(cancellationToken);
|
var healthy = await transport.CheckHealthAsync(cancellationToken);
|
||||||
var backend = transport.IsDurable ? "rabbitmq" : "memory";
|
var backend = transport.IsDurable ? "rabbitmq" : "memory";
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var backlog = await monitoring.GetSnapshotAsync(cancellationToken);
|
||||||
return healthy
|
return healthy
|
||||||
? Results.Ok(new { Status = "healthy", Backend = backend })
|
? Results.Ok(new
|
||||||
|
{
|
||||||
|
Status = "healthy",
|
||||||
|
Backend = backend,
|
||||||
|
Backlog = backlog
|
||||||
|
})
|
||||||
: Results.Json(
|
: Results.Json(
|
||||||
new { Status = "unhealthy", Backend = backend },
|
new
|
||||||
|
{
|
||||||
|
Status = "unhealthy",
|
||||||
|
Backend = backend,
|
||||||
|
Backlog = backlog
|
||||||
|
},
|
||||||
statusCode: StatusCodes.Status503ServiceUnavailable);
|
statusCode: StatusCodes.Status503ServiceUnavailable);
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
return Results.Json(
|
||||||
|
new
|
||||||
|
{
|
||||||
|
Status = "unhealthy",
|
||||||
|
Backend = backend,
|
||||||
|
Backlog = (object?)null
|
||||||
|
},
|
||||||
|
statusCode: StatusCodes.Status503ServiceUnavailable);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public partial class Program;
|
public partial class Program;
|
||||||
|
|||||||
@@ -24,10 +24,16 @@
|
|||||||
"PollIntervalMilliseconds": 500,
|
"PollIntervalMilliseconds": 500,
|
||||||
"LeaseSeconds": 120,
|
"LeaseSeconds": 120,
|
||||||
"PrefetchCount": 1,
|
"PrefetchCount": 1,
|
||||||
|
"AutomaticScheduleConcurrency": 1,
|
||||||
|
"SchedulePublishConcurrency": 1,
|
||||||
|
"MakeupExamAutoConcurrency": 1,
|
||||||
"Exchange": "jiaowu.background-jobs",
|
"Exchange": "jiaowu.background-jobs",
|
||||||
"QueuePrefix": "jiaowu.background-jobs",
|
"QueuePrefix": "jiaowu.background-jobs",
|
||||||
"UseQuorumQueues": true,
|
"UseQuorumQueues": true,
|
||||||
"ProcessingAttemptLimit": 5
|
"ProcessingAttemptLimit": 5,
|
||||||
|
"MaintenanceIntervalSeconds": 60,
|
||||||
|
"CompletedRetentionDays": 14,
|
||||||
|
"CleanupBatchSize": 500
|
||||||
},
|
},
|
||||||
"RabbitMq": {
|
"RabbitMq": {
|
||||||
"HostName": "localhost",
|
"HostName": "localhost",
|
||||||
|
|||||||
@@ -30,6 +30,14 @@ public sealed class AutomaticScheduleGeneratorTests
|
|||||||
Assert.True(await db.ScheduleTimeSlots.CountAsync() == 0);
|
Assert.True(await db.ScheduleTimeSlots.CountAsync() == 0);
|
||||||
Assert.True(await db.AutomaticScheduleJobs.CountAsync() == 0);
|
Assert.True(await db.AutomaticScheduleJobs.CountAsync() == 0);
|
||||||
Assert.True(await db.BackgroundJobOutboxMessages.CountAsync() == 0);
|
Assert.True(await db.BackgroundJobOutboxMessages.CountAsync() == 0);
|
||||||
|
Assert.True(await db.Database
|
||||||
|
.SqlQueryRaw<int>(
|
||||||
|
"""
|
||||||
|
SELECT COUNT(*) AS "Value"
|
||||||
|
FROM pragma_index_list('BackgroundJobOutboxMessages')
|
||||||
|
WHERE name = 'IX_BackgroundJobOutboxMessages_State_CompletedAt'
|
||||||
|
""")
|
||||||
|
.AnyAsync(value => value > 0));
|
||||||
Assert.True(await db.Database
|
Assert.True(await db.Database
|
||||||
.SqlQueryRaw<int>(
|
.SqlQueryRaw<int>(
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -10,7 +10,36 @@ namespace Jiaowu.Api.Tests;
|
|||||||
public sealed class BackgroundJobOutboxTests
|
public sealed class BackgroundJobOutboxTests
|
||||||
{
|
{
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task Publisher_moves_pending_message_to_in_memory_transport()
|
public async Task In_memory_transport_keeps_job_types_on_independent_channels()
|
||||||
|
{
|
||||||
|
var transport = new InMemoryBackgroundJobTransport();
|
||||||
|
var automatic = new BackgroundJobEnvelope(
|
||||||
|
Guid.NewGuid(),
|
||||||
|
BackgroundJobKind.AutomaticSchedule,
|
||||||
|
Guid.NewGuid());
|
||||||
|
var publish = new BackgroundJobEnvelope(
|
||||||
|
Guid.NewGuid(),
|
||||||
|
BackgroundJobKind.SchedulePublish,
|
||||||
|
Guid.NewGuid());
|
||||||
|
await transport.PublishAsync(automatic, CancellationToken.None);
|
||||||
|
await transport.PublishAsync(publish, CancellationToken.None);
|
||||||
|
|
||||||
|
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(1));
|
||||||
|
await using var automaticReader = transport
|
||||||
|
.ReadAllAsync(BackgroundJobKind.AutomaticSchedule, timeout.Token)
|
||||||
|
.GetAsyncEnumerator(timeout.Token);
|
||||||
|
await using var publishReader = transport
|
||||||
|
.ReadAllAsync(BackgroundJobKind.SchedulePublish, timeout.Token)
|
||||||
|
.GetAsyncEnumerator(timeout.Token);
|
||||||
|
|
||||||
|
Assert.True(await automaticReader.MoveNextAsync());
|
||||||
|
Assert.Equal(automatic, automaticReader.Current);
|
||||||
|
Assert.True(await publishReader.MoveNextAsync());
|
||||||
|
Assert.Equal(publish, publishReader.Current);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Publisher_recovers_pending_message_and_cleans_old_completion()
|
||||||
{
|
{
|
||||||
var databasePath = Path.Combine(
|
var databasePath = Path.Combine(
|
||||||
Path.GetTempPath(),
|
Path.GetTempPath(),
|
||||||
@@ -21,18 +50,22 @@ public sealed class BackgroundJobOutboxTests
|
|||||||
options.UseSqlite($"Data Source={databasePath};Pooling=False"));
|
options.UseSqlite($"Data Source={databasePath};Pooling=False"));
|
||||||
var options = new BackgroundJobOptions
|
var options = new BackgroundJobOptions
|
||||||
{
|
{
|
||||||
PollIntervalMilliseconds = 100
|
PollIntervalMilliseconds = 100,
|
||||||
|
CompletedRetentionDays = 1,
|
||||||
|
CleanupBatchSize = 10
|
||||||
};
|
};
|
||||||
services.AddSingleton(options);
|
services.AddSingleton(options);
|
||||||
services.AddSingleton<InMemoryBackgroundJobTransport>();
|
services.AddSingleton<InMemoryBackgroundJobTransport>();
|
||||||
services.AddSingleton<IBackgroundJobTransport>(provider =>
|
services.AddSingleton<IBackgroundJobTransport>(provider =>
|
||||||
provider.GetRequiredService<InMemoryBackgroundJobTransport>());
|
provider.GetRequiredService<InMemoryBackgroundJobTransport>());
|
||||||
|
services.AddSingleton<BackgroundJobTelemetry>();
|
||||||
services.AddSingleton<BackgroundJobOutboxPublisher>();
|
services.AddSingleton<BackgroundJobOutboxPublisher>();
|
||||||
var provider = services.BuildServiceProvider();
|
var provider = services.BuildServiceProvider();
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
Guid jobId;
|
Guid jobId;
|
||||||
|
Guid oldCompletedOutboxId;
|
||||||
await using (var scope = provider.CreateAsyncScope())
|
await using (var scope = provider.CreateAsyncScope())
|
||||||
{
|
{
|
||||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||||
@@ -57,8 +90,14 @@ public sealed class BackgroundJobOutboxTests
|
|||||||
SchedulePlan = plan,
|
SchedulePlan = plan,
|
||||||
ActiveSchedulePlanId = plan.Id
|
ActiveSchedulePlanId = plan.Id
|
||||||
};
|
};
|
||||||
|
var oldCompletedOutbox = BackgroundJobOutboxMessage.Create(
|
||||||
|
BackgroundJobKind.SchedulePublish,
|
||||||
|
Guid.NewGuid());
|
||||||
|
oldCompletedOutbox.State = BackgroundJobOutboxState.Completed;
|
||||||
|
oldCompletedOutbox.CompletedAt = DateTime.UtcNow.AddDays(-2);
|
||||||
|
oldCompletedOutboxId = oldCompletedOutbox.Id;
|
||||||
jobId = job.Id;
|
jobId = job.Id;
|
||||||
db.AddRange(term, plan, job);
|
db.AddRange(term, plan, job, oldCompletedOutbox);
|
||||||
await db.SaveChangesAsync();
|
await db.SaveChangesAsync();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -76,7 +115,9 @@ public sealed class BackgroundJobOutboxTests
|
|||||||
.AnyAsync(x =>
|
.AnyAsync(x =>
|
||||||
x.JobId == jobId &&
|
x.JobId == jobId &&
|
||||||
x.JobKind == BackgroundJobKind.AutomaticSchedule &&
|
x.JobKind == BackgroundJobKind.AutomaticSchedule &&
|
||||||
x.State == BackgroundJobOutboxState.Published);
|
x.State == BackgroundJobOutboxState.Published) &&
|
||||||
|
!await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||||
|
.AnyAsync(x => x.Id == oldCompletedOutboxId);
|
||||||
}
|
}
|
||||||
|
|
||||||
Assert.True(published);
|
Assert.True(published);
|
||||||
@@ -93,6 +134,47 @@ public sealed class BackgroundJobOutboxTests
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Monitoring_snapshot_reports_backlog_and_expired_leases()
|
||||||
|
{
|
||||||
|
var databasePath = Path.Combine(
|
||||||
|
Path.GetTempPath(),
|
||||||
|
$"jiaowu-monitoring-{Guid.NewGuid():N}.sqlite");
|
||||||
|
var options = new DbContextOptionsBuilder<AppDbContext>()
|
||||||
|
.UseSqlite($"Data Source={databasePath};Pooling=False")
|
||||||
|
.Options;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await using var db = new AppDbContext(options);
|
||||||
|
await db.Database.EnsureCreatedAsync();
|
||||||
|
var pending = BackgroundJobOutboxMessage.Create(
|
||||||
|
BackgroundJobKind.AutomaticSchedule,
|
||||||
|
Guid.NewGuid());
|
||||||
|
pending.CreatedAt = DateTime.UtcNow.AddMinutes(-2);
|
||||||
|
var processing = BackgroundJobOutboxMessage.Create(
|
||||||
|
BackgroundJobKind.MakeupExamAuto,
|
||||||
|
Guid.NewGuid());
|
||||||
|
processing.State = BackgroundJobOutboxState.Processing;
|
||||||
|
processing.LeaseExpiresAt = DateTime.UtcNow.AddMinutes(-1);
|
||||||
|
db.AddRange(pending, processing);
|
||||||
|
await db.SaveChangesAsync();
|
||||||
|
|
||||||
|
var snapshot = await new BackgroundJobMonitoringService(db)
|
||||||
|
.GetSnapshotAsync(CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Equal(1, snapshot.Pending);
|
||||||
|
Assert.Equal(1, snapshot.Processing);
|
||||||
|
Assert.Equal(1, snapshot.ExpiredLeases);
|
||||||
|
Assert.NotNull(snapshot.OldestUnfinishedAt);
|
||||||
|
Assert.True(snapshot.OldestUnfinishedAgeSeconds >= 100);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
File.Delete(databasePath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task Runner_stops_after_retry_limit_and_ignores_duplicate_delivery()
|
public async Task Runner_stops_after_retry_limit_and_ignores_duplicate_delivery()
|
||||||
{
|
{
|
||||||
@@ -109,6 +191,7 @@ public sealed class BackgroundJobOutboxTests
|
|||||||
services.AddSingleton<InMemoryBackgroundJobTransport>();
|
services.AddSingleton<InMemoryBackgroundJobTransport>();
|
||||||
services.AddSingleton<IBackgroundJobTransport>(provider =>
|
services.AddSingleton<IBackgroundJobTransport>(provider =>
|
||||||
provider.GetRequiredService<InMemoryBackgroundJobTransport>());
|
provider.GetRequiredService<InMemoryBackgroundJobTransport>());
|
||||||
|
services.AddSingleton<BackgroundJobTelemetry>();
|
||||||
services.AddSingleton<BackgroundJobRunner>();
|
services.AddSingleton<BackgroundJobRunner>();
|
||||||
var provider = services.BuildServiceProvider();
|
var provider = services.BuildServiceProvider();
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user