排课计算、课表发布、补考自动编排三类任务统一接入消息系统。
业务任务与 Outbox 消息同一事务落库,避免“数据库成功但消息没发出去”。 RabbitMQ 使用持久化消息、发布确认、手动 ACK、Quorum Queue、死信队列。 增加处理租约、心跳、异常重试、最大重试次数和幂等状态控制。 保留 InMemory 模式,开发环境无需安装 RabbitMQ。 支持多实例竞争消费,后续可以横向扩容。 新增 /health/messaging 消息系统健康检查。
This commit is contained in:
@@ -8,6 +8,9 @@ MYSQL_USER=jiaowu
|
|||||||
MYSQL_PASSWORD=
|
MYSQL_PASSWORD=
|
||||||
MYSQL_ROOT_PASSWORD=
|
MYSQL_ROOT_PASSWORD=
|
||||||
|
|
||||||
|
RABBITMQ_USER=jiaowu
|
||||||
|
RABBITMQ_PASSWORD=
|
||||||
|
|
||||||
JWT_KEY=
|
JWT_KEY=
|
||||||
ALLOWED_HOSTS=localhost
|
ALLOWED_HOSTS=localhost
|
||||||
CORS_ORIGIN=http://localhost:8080
|
CORS_ORIGIN=http://localhost:8080
|
||||||
|
|||||||
@@ -9,6 +9,16 @@ ConnectionStrings__MySql="Server=db.example.edu.cn;Port=3306;Database=jiaowu;Use
|
|||||||
# Redis 是可选加速器;留空时应用仅使用进程内缓存。
|
# Redis 是可选加速器;留空时应用仅使用进程内缓存。
|
||||||
# ConnectionStrings__Redis="redis.example.edu.cn:6380,user=jiaowu,password=REPLACE_WITH_A_STRONG_PASSWORD,ssl=true,abortConnect=false"
|
# ConnectionStrings__Redis="redis.example.edu.cn:6380,user=jiaowu,password=REPLACE_WITH_A_STRONG_PASSWORD,ssl=true,abortConnect=false"
|
||||||
|
|
||||||
|
# 单机可继续使用 InMemory;多实例生产部署建议启用 RabbitMQ。
|
||||||
|
BackgroundJobs__Transport=InMemory
|
||||||
|
# RabbitMq__HostName=rabbitmq.example.edu.cn
|
||||||
|
# RabbitMq__Port=5671
|
||||||
|
# RabbitMq__UserName=jiaowu
|
||||||
|
# RabbitMq__Password=REPLACE_WITH_A_STRONG_PASSWORD
|
||||||
|
# RabbitMq__VirtualHost=/jiaowu
|
||||||
|
# RabbitMq__UseTls=true
|
||||||
|
# RabbitMq__TlsServerName=rabbitmq.example.edu.cn
|
||||||
|
|
||||||
Cache__Enabled=true
|
Cache__Enabled=true
|
||||||
Cache__KeyPrefix=jiaowu:v1
|
Cache__KeyPrefix=jiaowu:v1
|
||||||
Cache__ReferenceExpirationMinutes=30
|
Cache__ReferenceExpirationMinutes=30
|
||||||
|
|||||||
@@ -223,6 +223,8 @@ SQLite 只用于本地开发:新库通过 `EnsureCreated` 建立,已有开
|
|||||||
- `/health`、`/health/ready`:实际检查数据库连接,失败时返回 HTTP 503。
|
- `/health`、`/health/ready`:实际检查数据库连接,失败时返回 HTTP 503。
|
||||||
- `/health/cache`:检查可选 Redis;未配置 Redis 时返回 `disabled`,Redis
|
- `/health/cache`:检查可选 Redis;未配置 Redis 时返回 `disabled`,Redis
|
||||||
故障不会影响数据库就绪探针。
|
故障不会影响数据库就绪探针。
|
||||||
|
- `/health/messaging`:检查后台任务传输;单机内存队列返回 `memory`,启用
|
||||||
|
RabbitMQ 时实际检查代理连接。
|
||||||
|
|
||||||
### 查询缓存与 Redis
|
### 查询缓存与 Redis
|
||||||
|
|
||||||
@@ -251,6 +253,37 @@ Redis 只作为可丢弃的查询缓存。连接失败时应用回源数据库
|
|||||||
`allkeys-lfu` 淘汰策略,不启用持久化。`compose.app.example.yml` 不创建 Redis;
|
`allkeys-lfu` 淘汰策略,不启用持久化。`compose.app.example.yml` 不创建 Redis;
|
||||||
如需连接外部 Redis,在 `.env` 中配置上述连接串即可。
|
如需连接外部 Redis,在 `.env` 中配置上述连接串即可。
|
||||||
|
|
||||||
|
### 后台任务与 RabbitMQ
|
||||||
|
|
||||||
|
自动排课、课表发布和补考自动生成使用数据库 Outbox 保存任务消息。创建业务任务与
|
||||||
|
Outbox 消息在同一次 MySQL 提交中完成,后台发布器再将消息投递给任务 Worker;重复
|
||||||
|
投递通过 Outbox 处理租约和唯一任务键抑制。任务状态表仍是前端查询进度与错误信息的
|
||||||
|
唯一来源。
|
||||||
|
|
||||||
|
开发和单机部署默认使用有界进程内队列,不需要 RabbitMQ:
|
||||||
|
|
||||||
|
```text
|
||||||
|
BackgroundJobs__Transport=InMemory
|
||||||
|
```
|
||||||
|
|
||||||
|
多实例生产部署应切换为 RabbitMQ,并配置独立账号、虚拟主机和 TLS:
|
||||||
|
|
||||||
|
```text
|
||||||
|
BackgroundJobs__Transport=RabbitMq
|
||||||
|
RabbitMq__HostName=rabbitmq.example.edu.cn
|
||||||
|
RabbitMq__Port=5671
|
||||||
|
RabbitMq__UserName=jiaowu
|
||||||
|
RabbitMq__Password=REPLACE_WITH_A_STRONG_PASSWORD
|
||||||
|
RabbitMq__VirtualHost=/jiaowu
|
||||||
|
RabbitMq__UseTls=true
|
||||||
|
RabbitMq__TlsServerName=rabbitmq.example.edu.cn
|
||||||
|
```
|
||||||
|
|
||||||
|
RabbitMQ 传输使用持久消息、发布确认、手动消费确认、每种任务独立队列和死信队列。
|
||||||
|
默认创建 Quorum Queue,重任务的消费者预取数为 1。MySQL 或 RabbitMQ 暂时不可用时,
|
||||||
|
未完成消息会根据 Outbox 状态和租约继续补投,消费者必须保持业务处理幂等。迁移服务
|
||||||
|
应先应用 `BackgroundJobOutbox` 数据库迁移,再启动应用实例。
|
||||||
|
|
||||||
## 跨平台发布与 Docker
|
## 跨平台发布与 Docker
|
||||||
|
|
||||||
`.gitea/workflows/publish.yml` 只在推送 `v*` 标签或手动运行时执行,普通分支 push
|
`.gitea/workflows/publish.yml` 只在推送 `v*` 标签或手动运行时执行,普通分支 push
|
||||||
|
|||||||
@@ -16,6 +16,13 @@ x-jiaowu-environment: &jiaowu-environment
|
|||||||
ConnectionStrings__Redis: "redis:6379,abortConnect=false"
|
ConnectionStrings__Redis: "redis:6379,abortConnect=false"
|
||||||
Cache__Enabled: "true"
|
Cache__Enabled: "true"
|
||||||
Cache__KeyPrefix: "jiaowu:v1"
|
Cache__KeyPrefix: "jiaowu:v1"
|
||||||
|
BackgroundJobs__Transport: RabbitMq
|
||||||
|
RabbitMq__HostName: rabbitmq
|
||||||
|
RabbitMq__Port: "5672"
|
||||||
|
RabbitMq__UserName: "${RABBITMQ_USER:-jiaowu}"
|
||||||
|
RabbitMq__Password: "${RABBITMQ_PASSWORD:?请在 .env.docker 中设置 RABBITMQ_PASSWORD}"
|
||||||
|
RabbitMq__VirtualHost: "/"
|
||||||
|
RabbitMq__UseTls: "false"
|
||||||
Jwt__Issuer: Jiaowu.Api
|
Jwt__Issuer: Jiaowu.Api
|
||||||
Jwt__Audience: Jiaowu.Web
|
Jwt__Audience: Jiaowu.Web
|
||||||
Jwt__Key: "${JWT_KEY:?请在 .env.docker 中设置 JWT_KEY}"
|
Jwt__Key: "${JWT_KEY:?请在 .env.docker 中设置 JWT_KEY}"
|
||||||
@@ -34,6 +41,26 @@ x-json-logging: &json-logging
|
|||||||
max-file: "3"
|
max-file: "3"
|
||||||
|
|
||||||
services:
|
services:
|
||||||
|
rabbitmq:
|
||||||
|
image: rabbitmq:4.2-management-alpine
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
RABBITMQ_DEFAULT_USER: "${RABBITMQ_USER:-jiaowu}"
|
||||||
|
RABBITMQ_DEFAULT_PASS: "${RABBITMQ_PASSWORD:?请在 .env.docker 中设置 RABBITMQ_PASSWORD}"
|
||||||
|
healthcheck:
|
||||||
|
test:
|
||||||
|
- CMD
|
||||||
|
- rabbitmq-diagnostics
|
||||||
|
- -q
|
||||||
|
- ping
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 12
|
||||||
|
start_period: 20s
|
||||||
|
volumes:
|
||||||
|
- rabbitmq-data:/var/lib/rabbitmq
|
||||||
|
logging: *json-logging
|
||||||
|
|
||||||
redis:
|
redis:
|
||||||
image: redis:8.8-alpine
|
image: redis:8.8-alpine
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
@@ -96,6 +123,8 @@ services:
|
|||||||
<<: *jiaowu-image
|
<<: *jiaowu-image
|
||||||
environment: *jiaowu-environment
|
environment: *jiaowu-environment
|
||||||
depends_on:
|
depends_on:
|
||||||
|
rabbitmq:
|
||||||
|
condition: service_healthy
|
||||||
redis:
|
redis:
|
||||||
condition: service_started
|
condition: service_started
|
||||||
mysql:
|
mysql:
|
||||||
@@ -128,3 +157,4 @@ services:
|
|||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
mysql-data:
|
mysql-data:
|
||||||
|
rabbitmq-data:
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
using System.ComponentModel.DataAnnotations;
|
using System.ComponentModel.DataAnnotations;
|
||||||
using Jiaowu.Api.Domain.Academic;
|
using Jiaowu.Api.Domain.Academic;
|
||||||
using Jiaowu.Api.Domain.Identity;
|
using Jiaowu.Api.Domain.Identity;
|
||||||
|
using Jiaowu.Api.Domain.System;
|
||||||
using Jiaowu.Api.Infrastructure.Auth;
|
using Jiaowu.Api.Infrastructure.Auth;
|
||||||
using Jiaowu.Api.Infrastructure.Exams;
|
using Jiaowu.Api.Infrastructure.Exams;
|
||||||
using Jiaowu.Api.Infrastructure.Grades;
|
using Jiaowu.Api.Infrastructure.Grades;
|
||||||
@@ -18,8 +19,7 @@ public sealed class MakeupExamsController(
|
|||||||
AppDbContext db,
|
AppDbContext db,
|
||||||
ICurrentUserDataScope currentUserDataScope,
|
ICurrentUserDataScope currentUserDataScope,
|
||||||
MakeupExamEligibilityService eligibilityService,
|
MakeupExamEligibilityService eligibilityService,
|
||||||
MakeupExamArrangementService arrangementService,
|
MakeupExamArrangementService arrangementService) : ControllerBase
|
||||||
MakeupExamAutoJobQueue autoJobQueue) : ControllerBase
|
|
||||||
{
|
{
|
||||||
private const string Managers =
|
private const string Managers =
|
||||||
SystemRoles.SuperAdmin + "," + SystemRoles.AcademicAdmin;
|
SystemRoles.SuperAdmin + "," + SystemRoles.AcademicAdmin;
|
||||||
@@ -336,10 +336,12 @@ public sealed class MakeupExamsController(
|
|||||||
MakeupExamPlanId = planId
|
MakeupExamPlanId = planId
|
||||||
};
|
};
|
||||||
db.MakeupExamAutoJobs.Add(job);
|
db.MakeupExamAutoJobs.Add(job);
|
||||||
|
db.BackgroundJobOutboxMessages.Add(
|
||||||
|
BackgroundJobOutboxMessage.Create(
|
||||||
|
BackgroundJobKind.MakeupExamAuto,
|
||||||
|
job.Id));
|
||||||
await db.SaveChangesAsync(cancellationToken);
|
await db.SaveChangesAsync(cancellationToken);
|
||||||
|
|
||||||
autoJobQueue.Enqueue(job.Id);
|
|
||||||
|
|
||||||
return AcceptedAtAction(nameof(GetAutoJob), new { jobId = job.Id },
|
return AcceptedAtAction(nameof(GetAutoJob), new { jobId = job.Id },
|
||||||
new { jobId = job.Id, status = job.Status.ToString() });
|
new { jobId = job.Id, status = job.Status.ToString() });
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ using System.Security.Claims;
|
|||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using Jiaowu.Api.Domain.Academic;
|
using Jiaowu.Api.Domain.Academic;
|
||||||
using Jiaowu.Api.Domain.Identity;
|
using Jiaowu.Api.Domain.Identity;
|
||||||
|
using Jiaowu.Api.Domain.System;
|
||||||
using Jiaowu.Api.Infrastructure.Persistence;
|
using Jiaowu.Api.Infrastructure.Persistence;
|
||||||
using Jiaowu.Api.Infrastructure.Scheduling;
|
using Jiaowu.Api.Infrastructure.Scheduling;
|
||||||
using Microsoft.AspNetCore.Authorization;
|
using Microsoft.AspNetCore.Authorization;
|
||||||
@@ -15,9 +16,7 @@ namespace Jiaowu.Api.Controllers;
|
|||||||
[Authorize(Roles = ManagementRoles)]
|
[Authorize(Roles = ManagementRoles)]
|
||||||
[Route("api/schedules")]
|
[Route("api/schedules")]
|
||||||
public sealed class SchedulesController(
|
public sealed class SchedulesController(
|
||||||
AppDbContext db,
|
AppDbContext db) : ControllerBase
|
||||||
AutomaticScheduleJobQueue automaticScheduleJobQueue,
|
|
||||||
SchedulePublishJobQueue schedulePublishJobQueue) : ControllerBase
|
|
||||||
{
|
{
|
||||||
private const string ManagementRoles =
|
private const string ManagementRoles =
|
||||||
SystemRoles.SuperAdmin + "," +
|
SystemRoles.SuperAdmin + "," +
|
||||||
@@ -241,6 +240,10 @@ public sealed class SchedulesController(
|
|||||||
CurrentStep = "等待后台检查"
|
CurrentStep = "等待后台检查"
|
||||||
};
|
};
|
||||||
db.SchedulePublishJobs.Add(job);
|
db.SchedulePublishJobs.Add(job);
|
||||||
|
db.BackgroundJobOutboxMessages.Add(
|
||||||
|
BackgroundJobOutboxMessage.Create(
|
||||||
|
BackgroundJobKind.SchedulePublish,
|
||||||
|
job.Id));
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
await db.SaveChangesAsync(cancellationToken);
|
await db.SaveChangesAsync(cancellationToken);
|
||||||
@@ -263,7 +266,6 @@ public sealed class SchedulesController(
|
|||||||
ToResponse(existing));
|
ToResponse(existing));
|
||||||
}
|
}
|
||||||
|
|
||||||
schedulePublishJobQueue.Enqueue(job.Id);
|
|
||||||
return AcceptedAtAction(
|
return AcceptedAtAction(
|
||||||
nameof(GetSchedulePublishJob),
|
nameof(GetSchedulePublishJob),
|
||||||
new { jobId = job.Id },
|
new { jobId = job.Id },
|
||||||
@@ -321,6 +323,10 @@ public sealed class SchedulesController(
|
|||||||
RequestedByUserId = requestedByUserId
|
RequestedByUserId = requestedByUserId
|
||||||
};
|
};
|
||||||
db.AutomaticScheduleJobs.Add(job);
|
db.AutomaticScheduleJobs.Add(job);
|
||||||
|
db.BackgroundJobOutboxMessages.Add(
|
||||||
|
BackgroundJobOutboxMessage.Create(
|
||||||
|
BackgroundJobKind.AutomaticSchedule,
|
||||||
|
job.Id));
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
await db.SaveChangesAsync(cancellationToken);
|
await db.SaveChangesAsync(cancellationToken);
|
||||||
@@ -339,7 +345,6 @@ public sealed class SchedulesController(
|
|||||||
ToResponse(existing));
|
ToResponse(existing));
|
||||||
}
|
}
|
||||||
|
|
||||||
automaticScheduleJobQueue.Enqueue(job.Id);
|
|
||||||
return AcceptedAtAction(
|
return AcceptedAtAction(
|
||||||
nameof(GetAutomaticScheduleJob),
|
nameof(GetAutomaticScheduleJob),
|
||||||
new { jobId = job.Id },
|
new { jobId = job.Id },
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
using Jiaowu.Api.Domain.Common;
|
||||||
|
|
||||||
|
namespace Jiaowu.Api.Domain.System;
|
||||||
|
|
||||||
|
public sealed class BackgroundJobOutboxMessage : EntityBase
|
||||||
|
{
|
||||||
|
public BackgroundJobKind JobKind { get; set; }
|
||||||
|
public Guid JobId { get; set; }
|
||||||
|
public BackgroundJobOutboxState State { get; set; } =
|
||||||
|
BackgroundJobOutboxState.Pending;
|
||||||
|
public int PublishAttempts { get; set; }
|
||||||
|
public int ProcessingAttempts { get; set; }
|
||||||
|
public DateTime? PublishedAt { get; set; }
|
||||||
|
public Guid? ProcessingToken { get; set; }
|
||||||
|
public DateTime? LeaseExpiresAt { get; set; }
|
||||||
|
public DateTime? CompletedAt { get; set; }
|
||||||
|
public string? LastError { get; set; }
|
||||||
|
|
||||||
|
public static BackgroundJobOutboxMessage Create(
|
||||||
|
BackgroundJobKind jobKind,
|
||||||
|
Guid jobId) =>
|
||||||
|
new()
|
||||||
|
{
|
||||||
|
JobKind = jobKind,
|
||||||
|
JobId = jobId
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum BackgroundJobKind
|
||||||
|
{
|
||||||
|
AutomaticSchedule = 1,
|
||||||
|
SchedulePublish = 2,
|
||||||
|
MakeupExamAuto = 3
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum BackgroundJobOutboxState
|
||||||
|
{
|
||||||
|
Pending = 1,
|
||||||
|
Publishing = 2,
|
||||||
|
Published = 3,
|
||||||
|
Processing = 4,
|
||||||
|
Completed = 5
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||||
|
|
||||||
|
public sealed class BackgroundJobOptions
|
||||||
|
{
|
||||||
|
public const string SectionName = "BackgroundJobs";
|
||||||
|
|
||||||
|
public string Transport { get; set; } = "InMemory";
|
||||||
|
public int PollIntervalMilliseconds { get; set; } = 500;
|
||||||
|
public int LeaseSeconds { get; set; } = 120;
|
||||||
|
public ushort PrefetchCount { get; set; } = 1;
|
||||||
|
public string Exchange { get; set; } = "jiaowu.background-jobs";
|
||||||
|
public string QueuePrefix { get; set; } = "jiaowu.background-jobs";
|
||||||
|
public bool UseQuorumQueues { get; set; } = true;
|
||||||
|
public int ProcessingAttemptLimit { get; set; } = 5;
|
||||||
|
|
||||||
|
public bool UsesRabbitMq =>
|
||||||
|
Transport.Equals("RabbitMq", StringComparison.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class RabbitMqOptions
|
||||||
|
{
|
||||||
|
public const string SectionName = "RabbitMq";
|
||||||
|
|
||||||
|
public string HostName { get; set; } = "localhost";
|
||||||
|
public int Port { get; set; } = 5672;
|
||||||
|
public string UserName { get; set; } = "guest";
|
||||||
|
public string Password { get; set; } = "guest";
|
||||||
|
public string VirtualHost { get; set; } = "/";
|
||||||
|
public bool UseTls { get; set; }
|
||||||
|
public string? TlsServerName { get; set; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,219 @@
|
|||||||
|
using Jiaowu.Api.Domain.Academic;
|
||||||
|
using Jiaowu.Api.Domain.System;
|
||||||
|
using Jiaowu.Api.Infrastructure.Persistence;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
|
||||||
|
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||||
|
|
||||||
|
public sealed class BackgroundJobOutboxPublisher(
|
||||||
|
IServiceScopeFactory scopeFactory,
|
||||||
|
IBackgroundJobTransport transport,
|
||||||
|
BackgroundJobOptions options,
|
||||||
|
ILogger<BackgroundJobOutboxPublisher> logger) : BackgroundService
|
||||||
|
{
|
||||||
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
|
{
|
||||||
|
await PrepareForStartupAsync(stoppingToken);
|
||||||
|
|
||||||
|
while (!stoppingToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await RecoverExpiredProcessingAsync(stoppingToken);
|
||||||
|
var claimed = await ClaimNextAsync(stoppingToken);
|
||||||
|
if (claimed is null)
|
||||||
|
{
|
||||||
|
await Task.Delay(options.PollIntervalMilliseconds, stoppingToken);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
await PublishClaimedAsync(claimed.Value.Message, claimed.Value.Token,
|
||||||
|
stoppingToken);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
logger.LogError(exception, "Background job outbox publishing failed.");
|
||||||
|
await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task PrepareForStartupAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await using var scope = scopeFactory.CreateAsyncScope();
|
||||||
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||||
|
|
||||||
|
if (!transport.IsDurable)
|
||||||
|
{
|
||||||
|
await db.BackgroundJobOutboxMessages
|
||||||
|
.Where(x => x.State != BackgroundJobOutboxState.Completed)
|
||||||
|
.ExecuteUpdateAsync(
|
||||||
|
setters => setters
|
||||||
|
.SetProperty(x => x.State, BackgroundJobOutboxState.Pending)
|
||||||
|
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||||
|
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null),
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
await AddMissingOutboxMessagesAsync(db, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task AddMissingOutboxMessagesAsync(
|
||||||
|
AppDbContext db,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var existing = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||||
|
.Select(x => new { x.JobKind, x.JobId })
|
||||||
|
.ToListAsync(cancellationToken);
|
||||||
|
var existingKeys = existing
|
||||||
|
.Select(x => (x.JobKind, x.JobId))
|
||||||
|
.ToHashSet();
|
||||||
|
|
||||||
|
var automaticJobs = await db.AutomaticScheduleJobs.AsNoTracking()
|
||||||
|
.Where(x => x.Status == AutomaticScheduleJobStatus.Queued ||
|
||||||
|
x.Status == AutomaticScheduleJobStatus.Running)
|
||||||
|
.Select(x => x.Id)
|
||||||
|
.ToListAsync(cancellationToken);
|
||||||
|
var publishJobs = await db.SchedulePublishJobs.AsNoTracking()
|
||||||
|
.Where(x => x.Status == SchedulePublishJobStatus.Queued ||
|
||||||
|
x.Status == SchedulePublishJobStatus.Running)
|
||||||
|
.Select(x => x.Id)
|
||||||
|
.ToListAsync(cancellationToken);
|
||||||
|
var makeupJobs = await db.MakeupExamAutoJobs.AsNoTracking()
|
||||||
|
.Where(x => x.Status == MakeupExamAutoJobStatus.Queued ||
|
||||||
|
x.Status == MakeupExamAutoJobStatus.Running)
|
||||||
|
.Select(x => x.Id)
|
||||||
|
.ToListAsync(cancellationToken);
|
||||||
|
|
||||||
|
AddMissing(BackgroundJobKind.AutomaticSchedule, automaticJobs);
|
||||||
|
AddMissing(BackgroundJobKind.SchedulePublish, publishJobs);
|
||||||
|
AddMissing(BackgroundJobKind.MakeupExamAuto, makeupJobs);
|
||||||
|
|
||||||
|
if (db.ChangeTracker.HasChanges())
|
||||||
|
await db.SaveChangesAsync(cancellationToken);
|
||||||
|
|
||||||
|
void AddMissing(BackgroundJobKind kind, IEnumerable<Guid> jobIds)
|
||||||
|
{
|
||||||
|
foreach (var jobId in jobIds)
|
||||||
|
{
|
||||||
|
if (existingKeys.Add((kind, jobId)))
|
||||||
|
{
|
||||||
|
db.BackgroundJobOutboxMessages.Add(
|
||||||
|
BackgroundJobOutboxMessage.Create(kind, jobId));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task RecoverExpiredProcessingAsync(
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await using var scope = scopeFactory.CreateAsyncScope();
|
||||||
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||||
|
var now = DateTime.UtcNow;
|
||||||
|
var recoveredState = transport.IsDurable
|
||||||
|
? BackgroundJobOutboxState.Published
|
||||||
|
: BackgroundJobOutboxState.Pending;
|
||||||
|
|
||||||
|
await db.BackgroundJobOutboxMessages
|
||||||
|
.Where(x =>
|
||||||
|
(x.State == BackgroundJobOutboxState.Publishing ||
|
||||||
|
x.State == BackgroundJobOutboxState.Processing) &&
|
||||||
|
x.LeaseExpiresAt != null &&
|
||||||
|
x.LeaseExpiresAt < now)
|
||||||
|
.ExecuteUpdateAsync(
|
||||||
|
setters => setters
|
||||||
|
.SetProperty(x => x.State, recoveredState)
|
||||||
|
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||||
|
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null),
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<(BackgroundJobEnvelope Message, Guid Token)?> ClaimNextAsync(
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await using var scope = scopeFactory.CreateAsyncScope();
|
||||||
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||||
|
var candidateId = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||||
|
.Where(x => x.State == BackgroundJobOutboxState.Pending)
|
||||||
|
.OrderBy(x => x.CreatedAt)
|
||||||
|
.Select(x => (Guid?)x.Id)
|
||||||
|
.FirstOrDefaultAsync(cancellationToken);
|
||||||
|
if (!candidateId.HasValue)
|
||||||
|
return null;
|
||||||
|
|
||||||
|
var token = Guid.NewGuid();
|
||||||
|
var leaseExpiresAt = DateTime.UtcNow.AddSeconds(options.LeaseSeconds);
|
||||||
|
var claimed = await db.BackgroundJobOutboxMessages
|
||||||
|
.Where(x =>
|
||||||
|
x.Id == candidateId.Value &&
|
||||||
|
x.State == BackgroundJobOutboxState.Pending)
|
||||||
|
.ExecuteUpdateAsync(
|
||||||
|
setters => setters
|
||||||
|
.SetProperty(x => x.State, BackgroundJobOutboxState.Publishing)
|
||||||
|
.SetProperty(x => x.ProcessingToken, token)
|
||||||
|
.SetProperty(x => x.LeaseExpiresAt, leaseExpiresAt)
|
||||||
|
.SetProperty(x => x.PublishAttempts, x => x.PublishAttempts + 1)
|
||||||
|
.SetProperty(x => x.LastError, (string?)null),
|
||||||
|
cancellationToken);
|
||||||
|
if (claimed == 0)
|
||||||
|
return null;
|
||||||
|
|
||||||
|
var message = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||||
|
.Where(x => x.Id == candidateId.Value)
|
||||||
|
.Select(x => new BackgroundJobEnvelope(x.Id, x.JobKind, x.JobId))
|
||||||
|
.SingleAsync(cancellationToken);
|
||||||
|
return (message, token);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task PublishClaimedAsync(
|
||||||
|
BackgroundJobEnvelope message,
|
||||||
|
Guid token,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await transport.PublishAsync(message, cancellationToken);
|
||||||
|
|
||||||
|
await using var scope = scopeFactory.CreateAsyncScope();
|
||||||
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||||
|
await db.BackgroundJobOutboxMessages
|
||||||
|
.Where(x =>
|
||||||
|
x.Id == message.OutboxMessageId &&
|
||||||
|
x.State == BackgroundJobOutboxState.Publishing &&
|
||||||
|
x.ProcessingToken == token)
|
||||||
|
.ExecuteUpdateAsync(
|
||||||
|
setters => setters
|
||||||
|
.SetProperty(x => x.State, BackgroundJobOutboxState.Published)
|
||||||
|
.SetProperty(x => x.PublishedAt, DateTime.UtcNow)
|
||||||
|
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||||
|
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null),
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
await using var scope = scopeFactory.CreateAsyncScope();
|
||||||
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||||
|
var messageText = exception.GetBaseException().Message;
|
||||||
|
if (messageText.Length > 2000)
|
||||||
|
messageText = messageText[..2000];
|
||||||
|
await db.BackgroundJobOutboxMessages
|
||||||
|
.Where(x =>
|
||||||
|
x.Id == message.OutboxMessageId &&
|
||||||
|
x.State == BackgroundJobOutboxState.Publishing &&
|
||||||
|
x.ProcessingToken == token)
|
||||||
|
.ExecuteUpdateAsync(
|
||||||
|
setters => setters
|
||||||
|
.SetProperty(x => x.State, BackgroundJobOutboxState.Pending)
|
||||||
|
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||||
|
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null)
|
||||||
|
.SetProperty(x => x.LastError, messageText),
|
||||||
|
cancellationToken);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,333 @@
|
|||||||
|
using Jiaowu.Api.Domain.Academic;
|
||||||
|
using Jiaowu.Api.Domain.System;
|
||||||
|
using Jiaowu.Api.Infrastructure.Exams;
|
||||||
|
using Jiaowu.Api.Infrastructure.Persistence;
|
||||||
|
using Jiaowu.Api.Infrastructure.Scheduling;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
|
||||||
|
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||||
|
|
||||||
|
public sealed record BackgroundJobRunResult(
|
||||||
|
BackgroundJobRunOutcome Outcome,
|
||||||
|
TimeSpan RetryAfter)
|
||||||
|
{
|
||||||
|
public static BackgroundJobRunResult Completed { get; } =
|
||||||
|
new(BackgroundJobRunOutcome.Completed, TimeSpan.Zero);
|
||||||
|
|
||||||
|
public static BackgroundJobRunResult Retry(TimeSpan retryAfter) =>
|
||||||
|
new(BackgroundJobRunOutcome.Retry, retryAfter);
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum BackgroundJobRunOutcome
|
||||||
|
{
|
||||||
|
Completed,
|
||||||
|
Retry
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class BackgroundJobRunner(
|
||||||
|
IServiceScopeFactory scopeFactory,
|
||||||
|
IBackgroundJobTransport transport,
|
||||||
|
BackgroundJobOptions options,
|
||||||
|
ILogger<BackgroundJobRunner> logger)
|
||||||
|
{
|
||||||
|
public async Task<BackgroundJobRunResult> RunAsync(
|
||||||
|
BackgroundJobEnvelope message,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var token = Guid.NewGuid();
|
||||||
|
var claim = await TryClaimAsync(message, token, cancellationToken);
|
||||||
|
if (!claim.Claimed)
|
||||||
|
{
|
||||||
|
return claim.Completed
|
||||||
|
? BackgroundJobRunResult.Completed
|
||||||
|
: BackgroundJobRunResult.Retry(claim.RetryAfter);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (claim.AttemptsExceeded)
|
||||||
|
{
|
||||||
|
await MarkJobRetryLimitExceededAsync(message, cancellationToken);
|
||||||
|
await MarkCompletedAsync(message.OutboxMessageId, token,
|
||||||
|
cancellationToken);
|
||||||
|
return BackgroundJobRunResult.Completed;
|
||||||
|
}
|
||||||
|
|
||||||
|
using var heartbeatCancellation =
|
||||||
|
CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||||
|
var heartbeat = RunHeartbeatAsync(
|
||||||
|
message.OutboxMessageId,
|
||||||
|
token,
|
||||||
|
heartbeatCancellation.Token);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await using var scope = scopeFactory.CreateAsyncScope();
|
||||||
|
switch (message.JobKind)
|
||||||
|
{
|
||||||
|
case BackgroundJobKind.AutomaticSchedule:
|
||||||
|
await scope.ServiceProvider
|
||||||
|
.GetRequiredService<AutomaticScheduleJobProcessor>()
|
||||||
|
.ProcessAsync(message.JobId, cancellationToken);
|
||||||
|
break;
|
||||||
|
case BackgroundJobKind.SchedulePublish:
|
||||||
|
await scope.ServiceProvider
|
||||||
|
.GetRequiredService<SchedulePublishJobProcessor>()
|
||||||
|
.ProcessAsync(message.JobId, cancellationToken);
|
||||||
|
break;
|
||||||
|
case BackgroundJobKind.MakeupExamAuto:
|
||||||
|
await scope.ServiceProvider
|
||||||
|
.GetRequiredService<MakeupExamAutoJobProcessor>()
|
||||||
|
.ProcessAsync(message.JobId, cancellationToken);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"Unsupported background job kind '{message.JobKind}'.");
|
||||||
|
}
|
||||||
|
|
||||||
|
await MarkCompletedAsync(message.OutboxMessageId, token,
|
||||||
|
cancellationToken);
|
||||||
|
return BackgroundJobRunResult.Completed;
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
logger.LogError(
|
||||||
|
exception,
|
||||||
|
"Background job {JobKind}/{JobId} will be retried.",
|
||||||
|
message.JobKind,
|
||||||
|
message.JobId);
|
||||||
|
await ReleaseForRetryAsync(
|
||||||
|
message.OutboxMessageId,
|
||||||
|
token,
|
||||||
|
exception,
|
||||||
|
CancellationToken.None);
|
||||||
|
return BackgroundJobRunResult.Retry(TimeSpan.FromSeconds(2));
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
await heartbeatCancellation.CancelAsync();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await heartbeat;
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException)
|
||||||
|
{
|
||||||
|
// Expected when processing completes or the application stops.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<(
|
||||||
|
bool Claimed,
|
||||||
|
bool Completed,
|
||||||
|
bool AttemptsExceeded,
|
||||||
|
TimeSpan RetryAfter)>
|
||||||
|
TryClaimAsync(
|
||||||
|
BackgroundJobEnvelope message,
|
||||||
|
Guid token,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await using var scope = scopeFactory.CreateAsyncScope();
|
||||||
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||||
|
var now = DateTime.UtcNow;
|
||||||
|
var leaseExpiresAt = now.AddSeconds(options.LeaseSeconds);
|
||||||
|
var claimed = await db.BackgroundJobOutboxMessages
|
||||||
|
.Where(x =>
|
||||||
|
x.Id == message.OutboxMessageId &&
|
||||||
|
x.JobKind == message.JobKind &&
|
||||||
|
x.JobId == message.JobId &&
|
||||||
|
(x.State == BackgroundJobOutboxState.Pending ||
|
||||||
|
x.State == BackgroundJobOutboxState.Publishing ||
|
||||||
|
x.State == BackgroundJobOutboxState.Published ||
|
||||||
|
(x.State == BackgroundJobOutboxState.Processing &&
|
||||||
|
x.LeaseExpiresAt != null &&
|
||||||
|
x.LeaseExpiresAt < now)))
|
||||||
|
.ExecuteUpdateAsync(
|
||||||
|
setters => setters
|
||||||
|
.SetProperty(x => x.State, BackgroundJobOutboxState.Processing)
|
||||||
|
.SetProperty(x => x.ProcessingToken, token)
|
||||||
|
.SetProperty(x => x.LeaseExpiresAt, leaseExpiresAt)
|
||||||
|
.SetProperty(
|
||||||
|
x => x.ProcessingAttempts,
|
||||||
|
x => x.ProcessingAttempts + 1)
|
||||||
|
.SetProperty(x => x.LastError, (string?)null),
|
||||||
|
cancellationToken);
|
||||||
|
if (claimed == 1)
|
||||||
|
{
|
||||||
|
var attempts = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||||
|
.Where(x => x.Id == message.OutboxMessageId)
|
||||||
|
.Select(x => x.ProcessingAttempts)
|
||||||
|
.SingleAsync(cancellationToken);
|
||||||
|
return (
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
attempts > options.ProcessingAttemptLimit,
|
||||||
|
TimeSpan.Zero);
|
||||||
|
}
|
||||||
|
|
||||||
|
var current = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||||
|
.Where(x => x.Id == message.OutboxMessageId)
|
||||||
|
.Select(x => new { x.State, x.LeaseExpiresAt })
|
||||||
|
.FirstOrDefaultAsync(cancellationToken);
|
||||||
|
if (current is null || current.State == BackgroundJobOutboxState.Completed)
|
||||||
|
return (false, true, false, TimeSpan.Zero);
|
||||||
|
|
||||||
|
var retryAfter = current.LeaseExpiresAt.HasValue
|
||||||
|
? current.LeaseExpiresAt.Value - now
|
||||||
|
: TimeSpan.FromSeconds(2);
|
||||||
|
retryAfter = TimeSpan.FromSeconds(Math.Clamp(
|
||||||
|
retryAfter.TotalSeconds,
|
||||||
|
1,
|
||||||
|
options.LeaseSeconds));
|
||||||
|
return (false, false, false, retryAfter);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task MarkJobRetryLimitExceededAsync(
|
||||||
|
BackgroundJobEnvelope message,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await using var scope = scopeFactory.CreateAsyncScope();
|
||||||
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||||
|
var completedAt = DateTime.UtcNow;
|
||||||
|
var error = $"后台任务连续处理失败超过 {options.ProcessingAttemptLimit} 次," +
|
||||||
|
"已停止自动重试,请检查服务日志后重新创建任务。";
|
||||||
|
|
||||||
|
switch (message.JobKind)
|
||||||
|
{
|
||||||
|
case BackgroundJobKind.AutomaticSchedule:
|
||||||
|
await db.AutomaticScheduleJobs
|
||||||
|
.Where(x =>
|
||||||
|
x.Id == message.JobId &&
|
||||||
|
x.Status != AutomaticScheduleJobStatus.Succeeded &&
|
||||||
|
x.Status != AutomaticScheduleJobStatus.Failed)
|
||||||
|
.ExecuteUpdateAsync(
|
||||||
|
setters => setters
|
||||||
|
.SetProperty(
|
||||||
|
x => x.Status,
|
||||||
|
AutomaticScheduleJobStatus.Failed)
|
||||||
|
.SetProperty(x => x.ActiveSchedulePlanId, (Guid?)null)
|
||||||
|
.SetProperty(x => x.ErrorMessage, error)
|
||||||
|
.SetProperty(x => x.CompletedAt, completedAt),
|
||||||
|
cancellationToken);
|
||||||
|
break;
|
||||||
|
case BackgroundJobKind.SchedulePublish:
|
||||||
|
await db.SchedulePublishJobs
|
||||||
|
.Where(x =>
|
||||||
|
x.Id == message.JobId &&
|
||||||
|
x.Status != SchedulePublishJobStatus.Succeeded &&
|
||||||
|
x.Status != SchedulePublishJobStatus.Failed)
|
||||||
|
.ExecuteUpdateAsync(
|
||||||
|
setters => setters
|
||||||
|
.SetProperty(
|
||||||
|
x => x.Status,
|
||||||
|
SchedulePublishJobStatus.Failed)
|
||||||
|
.SetProperty(x => x.ActiveAcademicTermId, (Guid?)null)
|
||||||
|
.SetProperty(x => x.CurrentStep, "后台处理已停止")
|
||||||
|
.SetProperty(x => x.ErrorMessage, error)
|
||||||
|
.SetProperty(x => x.CompletedAt, completedAt),
|
||||||
|
cancellationToken);
|
||||||
|
break;
|
||||||
|
case BackgroundJobKind.MakeupExamAuto:
|
||||||
|
await db.MakeupExamAutoJobs
|
||||||
|
.Where(x =>
|
||||||
|
x.Id == message.JobId &&
|
||||||
|
x.Status != MakeupExamAutoJobStatus.Succeeded &&
|
||||||
|
x.Status != MakeupExamAutoJobStatus.Failed)
|
||||||
|
.ExecuteUpdateAsync(
|
||||||
|
setters => setters
|
||||||
|
.SetProperty(
|
||||||
|
x => x.Status,
|
||||||
|
MakeupExamAutoJobStatus.Failed)
|
||||||
|
.SetProperty(x => x.ErrorMessage, error)
|
||||||
|
.SetProperty(x => x.CompletedAt, completedAt),
|
||||||
|
cancellationToken);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new ArgumentOutOfRangeException(
|
||||||
|
nameof(message.JobKind),
|
||||||
|
message.JobKind,
|
||||||
|
null);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.LogError(
|
||||||
|
"Background job {JobKind}/{JobId} exceeded {AttemptLimit} processing attempts.",
|
||||||
|
message.JobKind,
|
||||||
|
message.JobId,
|
||||||
|
options.ProcessingAttemptLimit);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task RunHeartbeatAsync(
|
||||||
|
Guid outboxMessageId,
|
||||||
|
Guid token,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var intervalSeconds = Math.Max(5, options.LeaseSeconds / 3);
|
||||||
|
while (!cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
await Task.Delay(TimeSpan.FromSeconds(intervalSeconds), cancellationToken);
|
||||||
|
await using var scope = scopeFactory.CreateAsyncScope();
|
||||||
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||||
|
await db.BackgroundJobOutboxMessages
|
||||||
|
.Where(x =>
|
||||||
|
x.Id == outboxMessageId &&
|
||||||
|
x.State == BackgroundJobOutboxState.Processing &&
|
||||||
|
x.ProcessingToken == token)
|
||||||
|
.ExecuteUpdateAsync(
|
||||||
|
setters => setters.SetProperty(
|
||||||
|
x => x.LeaseExpiresAt,
|
||||||
|
DateTime.UtcNow.AddSeconds(options.LeaseSeconds)),
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task MarkCompletedAsync(
|
||||||
|
Guid outboxMessageId,
|
||||||
|
Guid token,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await using var scope = scopeFactory.CreateAsyncScope();
|
||||||
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||||
|
await db.BackgroundJobOutboxMessages
|
||||||
|
.Where(x =>
|
||||||
|
x.Id == outboxMessageId &&
|
||||||
|
x.State == BackgroundJobOutboxState.Processing &&
|
||||||
|
x.ProcessingToken == token)
|
||||||
|
.ExecuteUpdateAsync(
|
||||||
|
setters => setters
|
||||||
|
.SetProperty(x => x.State, BackgroundJobOutboxState.Completed)
|
||||||
|
.SetProperty(x => x.CompletedAt, DateTime.UtcNow)
|
||||||
|
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||||
|
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null),
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task ReleaseForRetryAsync(
|
||||||
|
Guid outboxMessageId,
|
||||||
|
Guid token,
|
||||||
|
Exception exception,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await using var scope = scopeFactory.CreateAsyncScope();
|
||||||
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||||
|
var message = exception.GetBaseException().Message;
|
||||||
|
if (message.Length > 2000)
|
||||||
|
message = message[..2000];
|
||||||
|
var state = transport.IsDurable
|
||||||
|
? BackgroundJobOutboxState.Published
|
||||||
|
: BackgroundJobOutboxState.Pending;
|
||||||
|
await db.BackgroundJobOutboxMessages
|
||||||
|
.Where(x =>
|
||||||
|
x.Id == outboxMessageId &&
|
||||||
|
x.State == BackgroundJobOutboxState.Processing &&
|
||||||
|
x.ProcessingToken == token)
|
||||||
|
.ExecuteUpdateAsync(
|
||||||
|
setters => setters
|
||||||
|
.SetProperty(x => x.State, state)
|
||||||
|
.SetProperty(x => x.ProcessingToken, (Guid?)null)
|
||||||
|
.SetProperty(x => x.LeaseExpiresAt, (DateTime?)null)
|
||||||
|
.SetProperty(x => x.LastError, message),
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
using System.Threading.Channels;
|
||||||
|
using Jiaowu.Api.Domain.System;
|
||||||
|
|
||||||
|
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||||
|
|
||||||
|
public sealed record BackgroundJobEnvelope(
|
||||||
|
Guid OutboxMessageId,
|
||||||
|
BackgroundJobKind JobKind,
|
||||||
|
Guid JobId);
|
||||||
|
|
||||||
|
public interface IBackgroundJobTransport
|
||||||
|
{
|
||||||
|
bool IsDurable { get; }
|
||||||
|
|
||||||
|
ValueTask PublishAsync(
|
||||||
|
BackgroundJobEnvelope message,
|
||||||
|
CancellationToken cancellationToken);
|
||||||
|
|
||||||
|
Task<bool> CheckHealthAsync(CancellationToken cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class InMemoryBackgroundJobTransport : IBackgroundJobTransport
|
||||||
|
{
|
||||||
|
private readonly Channel<BackgroundJobEnvelope> _channel =
|
||||||
|
Channel.CreateBounded<BackgroundJobEnvelope>(
|
||||||
|
new BoundedChannelOptions(256)
|
||||||
|
{
|
||||||
|
FullMode = BoundedChannelFullMode.Wait,
|
||||||
|
SingleReader = true,
|
||||||
|
SingleWriter = false
|
||||||
|
});
|
||||||
|
|
||||||
|
public bool IsDurable => false;
|
||||||
|
|
||||||
|
public ValueTask PublishAsync(
|
||||||
|
BackgroundJobEnvelope message,
|
||||||
|
CancellationToken cancellationToken) =>
|
||||||
|
_channel.Writer.WriteAsync(message, cancellationToken);
|
||||||
|
|
||||||
|
public Task<bool> CheckHealthAsync(CancellationToken cancellationToken) =>
|
||||||
|
Task.FromResult(true);
|
||||||
|
|
||||||
|
public IAsyncEnumerable<BackgroundJobEnvelope> ReadAllAsync(
|
||||||
|
CancellationToken cancellationToken) =>
|
||||||
|
_channel.Reader.ReadAllAsync(cancellationToken);
|
||||||
|
}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||||
|
|
||||||
|
public sealed class InMemoryBackgroundJobWorker(
|
||||||
|
InMemoryBackgroundJobTransport transport,
|
||||||
|
BackgroundJobRunner runner,
|
||||||
|
ILogger<InMemoryBackgroundJobWorker> logger) : BackgroundService
|
||||||
|
{
|
||||||
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await foreach (var message in transport.ReadAllAsync(stoppingToken))
|
||||||
|
{
|
||||||
|
var result = await runner.RunAsync(message, stoppingToken);
|
||||||
|
if (result.Outcome != BackgroundJobRunOutcome.Retry)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
await Task.Delay(result.RetryAfter, stoppingToken);
|
||||||
|
await transport.PublishAsync(message, stoppingToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
logger.LogInformation("In-memory background job worker is stopping.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,416 @@
|
|||||||
|
using System.Text.Json;
|
||||||
|
using Jiaowu.Api.Domain.System;
|
||||||
|
using RabbitMQ.Client;
|
||||||
|
using RabbitMQ.Client.Events;
|
||||||
|
|
||||||
|
namespace Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||||
|
|
||||||
|
public sealed class RabbitMqBackgroundJobTransport(
|
||||||
|
BackgroundJobOptions jobOptions,
|
||||||
|
RabbitMqOptions rabbitOptions,
|
||||||
|
ILogger<RabbitMqBackgroundJobTransport> logger)
|
||||||
|
: IBackgroundJobTransport, IAsyncDisposable
|
||||||
|
{
|
||||||
|
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||||
|
private IConnection? _connection;
|
||||||
|
private IChannel? _channel;
|
||||||
|
|
||||||
|
public bool IsDurable => true;
|
||||||
|
|
||||||
|
public async ValueTask PublishAsync(
|
||||||
|
BackgroundJobEnvelope message,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var body = JsonSerializer.SerializeToUtf8Bytes(message);
|
||||||
|
await _gate.WaitAsync(cancellationToken);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var channel = await GetChannelAsync(cancellationToken);
|
||||||
|
var properties = new BasicProperties
|
||||||
|
{
|
||||||
|
Persistent = true,
|
||||||
|
ContentType = "application/json",
|
||||||
|
MessageId = message.OutboxMessageId.ToString("D"),
|
||||||
|
Type = message.JobKind.ToString(),
|
||||||
|
AppId = "jiaowu-api"
|
||||||
|
};
|
||||||
|
await channel.BasicPublishAsync(
|
||||||
|
jobOptions.Exchange,
|
||||||
|
RabbitMqBackgroundJobTopology.RoutingKey(message.JobKind),
|
||||||
|
mandatory: true,
|
||||||
|
properties,
|
||||||
|
body,
|
||||||
|
cancellationToken);
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
await ResetConnectionAsync();
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_gate.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<bool> CheckHealthAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
await _gate.WaitAsync(cancellationToken);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var channel = await GetChannelAsync(cancellationToken);
|
||||||
|
return channel.IsOpen;
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
logger.LogWarning(exception, "RabbitMQ health check failed.");
|
||||||
|
await ResetConnectionAsync();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_gate.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<IChannel> GetChannelAsync(CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
if (_channel is { IsOpen: true })
|
||||||
|
return _channel;
|
||||||
|
|
||||||
|
await ResetConnectionAsync();
|
||||||
|
_connection = await RabbitMqBackgroundJobTopology.CreateConnectionAsync(
|
||||||
|
rabbitOptions,
|
||||||
|
"jiaowu-background-job-publisher",
|
||||||
|
cancellationToken);
|
||||||
|
_channel = await _connection.CreateChannelAsync(
|
||||||
|
new CreateChannelOptions(
|
||||||
|
publisherConfirmationsEnabled: true,
|
||||||
|
publisherConfirmationTrackingEnabled: true),
|
||||||
|
cancellationToken);
|
||||||
|
await RabbitMqBackgroundJobTopology.DeclareAsync(
|
||||||
|
_channel,
|
||||||
|
jobOptions,
|
||||||
|
cancellationToken);
|
||||||
|
return _channel;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task ResetConnectionAsync()
|
||||||
|
{
|
||||||
|
if (_channel is not null)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await _channel.DisposeAsync();
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// The broker may already have closed the channel.
|
||||||
|
}
|
||||||
|
_channel = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (_connection is not null)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await _connection.DisposeAsync();
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// The broker may already have closed the connection.
|
||||||
|
}
|
||||||
|
_connection = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async ValueTask DisposeAsync()
|
||||||
|
{
|
||||||
|
await _gate.WaitAsync();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await ResetConnectionAsync();
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_gate.Release();
|
||||||
|
_gate.Dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class RabbitMqBackgroundJobWorker(
|
||||||
|
BackgroundJobOptions jobOptions,
|
||||||
|
RabbitMqOptions rabbitOptions,
|
||||||
|
BackgroundJobRunner runner,
|
||||||
|
ILogger<RabbitMqBackgroundJobWorker> logger) : BackgroundService
|
||||||
|
{
|
||||||
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
|
{
|
||||||
|
while (!stoppingToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
IConnection? connection = null;
|
||||||
|
var channels = new List<IChannel>();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
connection = await RabbitMqBackgroundJobTopology.CreateConnectionAsync(
|
||||||
|
rabbitOptions,
|
||||||
|
"jiaowu-background-job-worker",
|
||||||
|
stoppingToken);
|
||||||
|
|
||||||
|
foreach (var kind in RabbitMqBackgroundJobTopology.JobKinds)
|
||||||
|
{
|
||||||
|
var channel = await connection.CreateChannelAsync(
|
||||||
|
new CreateChannelOptions(
|
||||||
|
publisherConfirmationsEnabled: false,
|
||||||
|
publisherConfirmationTrackingEnabled: false),
|
||||||
|
stoppingToken);
|
||||||
|
channels.Add(channel);
|
||||||
|
await RabbitMqBackgroundJobTopology.DeclareAsync(
|
||||||
|
channel,
|
||||||
|
jobOptions,
|
||||||
|
stoppingToken);
|
||||||
|
await channel.BasicQosAsync(
|
||||||
|
0,
|
||||||
|
jobOptions.PrefetchCount,
|
||||||
|
global: false,
|
||||||
|
stoppingToken);
|
||||||
|
|
||||||
|
var consumer = new AsyncEventingBasicConsumer(channel);
|
||||||
|
consumer.ReceivedAsync += async (_, eventArgs) =>
|
||||||
|
{
|
||||||
|
using var deliveryCancellation =
|
||||||
|
CancellationTokenSource.CreateLinkedTokenSource(
|
||||||
|
eventArgs.CancellationToken,
|
||||||
|
stoppingToken);
|
||||||
|
var deliveryToken = deliveryCancellation.Token;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var message = JsonSerializer.Deserialize<BackgroundJobEnvelope>(
|
||||||
|
eventArgs.Body.Span);
|
||||||
|
if (message is null || message.JobKind != kind)
|
||||||
|
{
|
||||||
|
await channel.BasicNackAsync(
|
||||||
|
eventArgs.DeliveryTag,
|
||||||
|
multiple: false,
|
||||||
|
requeue: false,
|
||||||
|
deliveryToken);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var result = await runner.RunAsync(
|
||||||
|
message,
|
||||||
|
deliveryToken);
|
||||||
|
if (result.Outcome == BackgroundJobRunOutcome.Completed)
|
||||||
|
{
|
||||||
|
await channel.BasicAckAsync(
|
||||||
|
eventArgs.DeliveryTag,
|
||||||
|
multiple: false,
|
||||||
|
deliveryToken);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await Task.Delay(
|
||||||
|
result.RetryAfter,
|
||||||
|
deliveryToken);
|
||||||
|
await channel.BasicNackAsync(
|
||||||
|
eventArgs.DeliveryTag,
|
||||||
|
multiple: false,
|
||||||
|
requeue: true,
|
||||||
|
deliveryToken);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException)
|
||||||
|
{
|
||||||
|
// Closing the channel requeues unacknowledged deliveries.
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
logger.LogError(
|
||||||
|
exception,
|
||||||
|
"RabbitMQ delivery for {JobKind} failed and will be requeued.",
|
||||||
|
kind);
|
||||||
|
if (channel.IsOpen)
|
||||||
|
{
|
||||||
|
await channel.BasicNackAsync(
|
||||||
|
eventArgs.DeliveryTag,
|
||||||
|
multiple: false,
|
||||||
|
requeue: true,
|
||||||
|
CancellationToken.None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
await channel.BasicConsumeAsync(
|
||||||
|
RabbitMqBackgroundJobTopology.QueueName(jobOptions, kind),
|
||||||
|
autoAck: false,
|
||||||
|
consumer,
|
||||||
|
stoppingToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.LogInformation(
|
||||||
|
"RabbitMQ background job consumers are connected to {HostName}:{Port}.",
|
||||||
|
rabbitOptions.HostName,
|
||||||
|
rabbitOptions.Port);
|
||||||
|
while (connection.IsOpen && !stoppingToken.IsCancellationRequested)
|
||||||
|
await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
logger.LogError(
|
||||||
|
exception,
|
||||||
|
"RabbitMQ background job consumer connection failed; retrying.");
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
foreach (var channel in channels)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await channel.DisposeAsync();
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// The connection may already have disposed its channels.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (connection is not null)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await connection.DisposeAsync();
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// The broker may already have closed the connection.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!stoppingToken.IsCancellationRequested)
|
||||||
|
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static class RabbitMqBackgroundJobTopology
|
||||||
|
{
|
||||||
|
public static readonly BackgroundJobKind[] JobKinds =
|
||||||
|
[
|
||||||
|
BackgroundJobKind.AutomaticSchedule,
|
||||||
|
BackgroundJobKind.SchedulePublish,
|
||||||
|
BackgroundJobKind.MakeupExamAuto
|
||||||
|
];
|
||||||
|
|
||||||
|
public static async Task<IConnection> CreateConnectionAsync(
|
||||||
|
RabbitMqOptions options,
|
||||||
|
string clientName,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var factory = new ConnectionFactory
|
||||||
|
{
|
||||||
|
HostName = options.HostName,
|
||||||
|
Port = options.Port,
|
||||||
|
UserName = options.UserName,
|
||||||
|
Password = options.Password,
|
||||||
|
VirtualHost = options.VirtualHost,
|
||||||
|
ClientProvidedName = clientName,
|
||||||
|
AutomaticRecoveryEnabled = false,
|
||||||
|
TopologyRecoveryEnabled = false,
|
||||||
|
RequestedHeartbeat = TimeSpan.FromSeconds(30),
|
||||||
|
ConsumerDispatchConcurrency = 1
|
||||||
|
};
|
||||||
|
if (options.UseTls)
|
||||||
|
{
|
||||||
|
factory.Ssl = new SslOption
|
||||||
|
{
|
||||||
|
Enabled = true,
|
||||||
|
ServerName = string.IsNullOrWhiteSpace(options.TlsServerName)
|
||||||
|
? options.HostName
|
||||||
|
: options.TlsServerName
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return await factory.CreateConnectionAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async Task DeclareAsync(
|
||||||
|
IChannel channel,
|
||||||
|
BackgroundJobOptions options,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var deadExchange = options.Exchange + ".dead";
|
||||||
|
await channel.ExchangeDeclareAsync(
|
||||||
|
options.Exchange,
|
||||||
|
ExchangeType.Direct,
|
||||||
|
durable: true,
|
||||||
|
autoDelete: false,
|
||||||
|
cancellationToken: cancellationToken);
|
||||||
|
await channel.ExchangeDeclareAsync(
|
||||||
|
deadExchange,
|
||||||
|
ExchangeType.Direct,
|
||||||
|
durable: true,
|
||||||
|
autoDelete: false,
|
||||||
|
cancellationToken: cancellationToken);
|
||||||
|
|
||||||
|
foreach (var kind in JobKinds)
|
||||||
|
{
|
||||||
|
var routingKey = RoutingKey(kind);
|
||||||
|
var queueName = QueueName(options, kind);
|
||||||
|
var deadQueueName = queueName + ".dead";
|
||||||
|
var queueArguments = QueueArguments(options);
|
||||||
|
queueArguments["x-dead-letter-exchange"] = deadExchange;
|
||||||
|
queueArguments["x-dead-letter-routing-key"] = routingKey;
|
||||||
|
await channel.QueueDeclareAsync(
|
||||||
|
queueName,
|
||||||
|
durable: true,
|
||||||
|
exclusive: false,
|
||||||
|
autoDelete: false,
|
||||||
|
arguments: queueArguments,
|
||||||
|
cancellationToken: cancellationToken);
|
||||||
|
await channel.QueueBindAsync(
|
||||||
|
queueName,
|
||||||
|
options.Exchange,
|
||||||
|
routingKey,
|
||||||
|
cancellationToken: cancellationToken);
|
||||||
|
|
||||||
|
await channel.QueueDeclareAsync(
|
||||||
|
deadQueueName,
|
||||||
|
durable: true,
|
||||||
|
exclusive: false,
|
||||||
|
autoDelete: false,
|
||||||
|
arguments: QueueArguments(options),
|
||||||
|
cancellationToken: cancellationToken);
|
||||||
|
await channel.QueueBindAsync(
|
||||||
|
deadQueueName,
|
||||||
|
deadExchange,
|
||||||
|
routingKey,
|
||||||
|
cancellationToken: cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string RoutingKey(BackgroundJobKind kind) => kind switch
|
||||||
|
{
|
||||||
|
BackgroundJobKind.AutomaticSchedule => "schedule.automatic",
|
||||||
|
BackgroundJobKind.SchedulePublish => "schedule.publish",
|
||||||
|
BackgroundJobKind.MakeupExamAuto => "makeup-exam.automatic",
|
||||||
|
_ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null)
|
||||||
|
};
|
||||||
|
|
||||||
|
public static string QueueName(
|
||||||
|
BackgroundJobOptions options,
|
||||||
|
BackgroundJobKind kind) =>
|
||||||
|
$"{options.QueuePrefix}.{RoutingKey(kind)}";
|
||||||
|
|
||||||
|
private static Dictionary<string, object?> QueueArguments(
|
||||||
|
BackgroundJobOptions options)
|
||||||
|
{
|
||||||
|
var arguments = new Dictionary<string, object?>();
|
||||||
|
if (options.UseQuorumQueues)
|
||||||
|
arguments["x-queue-type"] = "quorum";
|
||||||
|
return arguments;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,111 +1,10 @@
|
|||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using System.Threading.Channels;
|
|
||||||
using Jiaowu.Api.Domain.Academic;
|
using Jiaowu.Api.Domain.Academic;
|
||||||
using Jiaowu.Api.Infrastructure.Persistence;
|
using Jiaowu.Api.Infrastructure.Persistence;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
|
||||||
namespace Jiaowu.Api.Infrastructure.Exams;
|
namespace Jiaowu.Api.Infrastructure.Exams;
|
||||||
|
|
||||||
public sealed class MakeupExamAutoJobQueue
|
|
||||||
{
|
|
||||||
private readonly Channel<Guid> _channel = Channel.CreateUnbounded<Guid>(
|
|
||||||
new UnboundedChannelOptions
|
|
||||||
{
|
|
||||||
SingleReader = true,
|
|
||||||
SingleWriter = false
|
|
||||||
});
|
|
||||||
|
|
||||||
public void Enqueue(Guid jobId)
|
|
||||||
{
|
|
||||||
if (!_channel.Writer.TryWrite(jobId))
|
|
||||||
throw new InvalidOperationException("补考自动生成任务队列当前不可用。");
|
|
||||||
}
|
|
||||||
|
|
||||||
public IAsyncEnumerable<Guid> ReadAllAsync(CancellationToken cancellationToken) =>
|
|
||||||
_channel.Reader.ReadAllAsync(cancellationToken);
|
|
||||||
}
|
|
||||||
|
|
||||||
public sealed class MakeupExamAutoJobWorker(
|
|
||||||
IServiceScopeFactory scopeFactory,
|
|
||||||
MakeupExamAutoJobQueue queue,
|
|
||||||
ILogger<MakeupExamAutoJobWorker> logger) : BackgroundService
|
|
||||||
{
|
|
||||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
await RecoverInterruptedJobsAsync(stoppingToken);
|
|
||||||
}
|
|
||||||
catch (Exception exception)
|
|
||||||
{
|
|
||||||
logger.LogWarning(
|
|
||||||
exception,
|
|
||||||
"Could not recover makeup exam auto jobs (table may not exist yet).");
|
|
||||||
}
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
await foreach (var jobId in queue.ReadAllAsync(stoppingToken))
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
await using var scope = scopeFactory.CreateAsyncScope();
|
|
||||||
var processor = scope.ServiceProvider
|
|
||||||
.GetRequiredService<MakeupExamAutoJobProcessor>();
|
|
||||||
await processor.ProcessAsync(jobId, stoppingToken);
|
|
||||||
}
|
|
||||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
|
||||||
{
|
|
||||||
throw;
|
|
||||||
}
|
|
||||||
catch (Exception exception)
|
|
||||||
{
|
|
||||||
logger.LogError(
|
|
||||||
exception,
|
|
||||||
"Unexpected failure while dispatching makeup exam auto job {JobId}.",
|
|
||||||
jobId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
|
||||||
{
|
|
||||||
logger.LogInformation("Makeup exam auto job worker is stopping.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async Task RecoverInterruptedJobsAsync(CancellationToken cancellationToken)
|
|
||||||
{
|
|
||||||
await using var scope = scopeFactory.CreateAsyncScope();
|
|
||||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
|
||||||
var jobs = await db.MakeupExamAutoJobs
|
|
||||||
.Where(x =>
|
|
||||||
x.Status == MakeupExamAutoJobStatus.Queued ||
|
|
||||||
x.Status == MakeupExamAutoJobStatus.Running)
|
|
||||||
.OrderBy(x => x.CreatedAt)
|
|
||||||
.ToListAsync(cancellationToken);
|
|
||||||
|
|
||||||
foreach (var job in jobs)
|
|
||||||
{
|
|
||||||
job.Status = MakeupExamAutoJobStatus.Queued;
|
|
||||||
job.StartedAt = null;
|
|
||||||
job.CompletedAt = null;
|
|
||||||
job.ErrorMessage = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (jobs.Count > 0)
|
|
||||||
await db.SaveChangesAsync(cancellationToken);
|
|
||||||
foreach (var job in jobs)
|
|
||||||
queue.Enqueue(job.Id);
|
|
||||||
|
|
||||||
if (jobs.Count > 0)
|
|
||||||
{
|
|
||||||
logger.LogInformation(
|
|
||||||
"Recovered {JobCount} queued or interrupted makeup exam auto jobs.",
|
|
||||||
jobs.Count);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public sealed class MakeupExamAutoJobProcessor(
|
public sealed class MakeupExamAutoJobProcessor(
|
||||||
AppDbContext db,
|
AppDbContext db,
|
||||||
MakeupExamEligibilityService eligibilityService,
|
MakeupExamEligibilityService eligibilityService,
|
||||||
|
|||||||
@@ -98,6 +98,8 @@ public sealed class AppDbContext(DbContextOptions<AppDbContext> options)
|
|||||||
public DbSet<OfficialDocumentDownload> OfficialDocumentDownloads =>
|
public DbSet<OfficialDocumentDownload> OfficialDocumentDownloads =>
|
||||||
Set<OfficialDocumentDownload>();
|
Set<OfficialDocumentDownload>();
|
||||||
public DbSet<AuditLog> AuditLogs => Set<AuditLog>();
|
public DbSet<AuditLog> AuditLogs => Set<AuditLog>();
|
||||||
|
public DbSet<BackgroundJobOutboxMessage> BackgroundJobOutboxMessages =>
|
||||||
|
Set<BackgroundJobOutboxMessage>();
|
||||||
|
|
||||||
protected override void ConfigureConventions(
|
protected override void ConfigureConventions(
|
||||||
ModelConfigurationBuilder configurationBuilder)
|
ModelConfigurationBuilder configurationBuilder)
|
||||||
@@ -993,6 +995,14 @@ public sealed class AppDbContext(DbContextOptions<AppDbContext> options)
|
|||||||
entity.HasIndex(x => x.CreatedAt);
|
entity.HasIndex(x => x.CreatedAt);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
builder.Entity<BackgroundJobOutboxMessage>(entity =>
|
||||||
|
{
|
||||||
|
entity.Property(x => x.LastError).HasMaxLength(2000);
|
||||||
|
entity.HasIndex(x => new { x.JobKind, x.JobId }).IsUnique();
|
||||||
|
entity.HasIndex(x => new { x.State, x.CreatedAt });
|
||||||
|
entity.HasIndex(x => x.LeaseExpiresAt);
|
||||||
|
});
|
||||||
|
|
||||||
builder.Entity<OfficialDocument>(entity =>
|
builder.Entity<OfficialDocument>(entity =>
|
||||||
{
|
{
|
||||||
entity.Property(x => x.DocumentNumber).HasMaxLength(50);
|
entity.Property(x => x.DocumentNumber).HasMaxLength(50);
|
||||||
|
|||||||
@@ -58,6 +58,8 @@ public sealed class DevelopmentSqliteMigrator(
|
|||||||
"20260726_31_unified_message_center";
|
"20260726_31_unified_message_center";
|
||||||
private const string ClassroomReservationsMigration =
|
private const string ClassroomReservationsMigration =
|
||||||
"20260726_32_classroom_reservations";
|
"20260726_32_classroom_reservations";
|
||||||
|
private const string BackgroundJobOutboxMigration =
|
||||||
|
"20260726_33_background_job_outbox";
|
||||||
|
|
||||||
public async Task MigrateAsync(CancellationToken cancellationToken = default)
|
public async Task MigrateAsync(CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
@@ -421,6 +423,19 @@ public sealed class DevelopmentSqliteMigrator(
|
|||||||
ClassroomReservationsMigration,
|
ClassroomReservationsMigration,
|
||||||
classroomReservationsExist ? [] : ClassroomReservationStatements,
|
classroomReservationsExist ? [] : ClassroomReservationStatements,
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
|
|
||||||
|
var backgroundJobOutboxExists = await db.Database
|
||||||
|
.SqlQueryRaw<int>(
|
||||||
|
"""
|
||||||
|
SELECT COUNT(*) AS "Value"
|
||||||
|
FROM sqlite_master
|
||||||
|
WHERE type = 'table' AND name = 'BackgroundJobOutboxMessages'
|
||||||
|
""")
|
||||||
|
.AnyAsync(value => value > 0, cancellationToken);
|
||||||
|
await ApplyMigrationAsync(
|
||||||
|
BackgroundJobOutboxMigration,
|
||||||
|
backgroundJobOutboxExists ? [] : BackgroundJobOutboxStatements,
|
||||||
|
cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task ApplyMigrationAsync(
|
private async Task ApplyMigrationAsync(
|
||||||
@@ -2004,4 +2019,37 @@ public sealed class DevelopmentSqliteMigrator(
|
|||||||
ON "ClassroomReservations" ("ReviewedByUserId");
|
ON "ClassroomReservations" ("ReviewedByUserId");
|
||||||
"""
|
"""
|
||||||
];
|
];
|
||||||
|
|
||||||
|
private static readonly string[] BackgroundJobOutboxStatements =
|
||||||
|
[
|
||||||
|
"""
|
||||||
|
CREATE TABLE "BackgroundJobOutboxMessages" (
|
||||||
|
"Id" TEXT NOT NULL CONSTRAINT "PK_BackgroundJobOutboxMessages" PRIMARY KEY,
|
||||||
|
"JobKind" INTEGER NOT NULL,
|
||||||
|
"JobId" TEXT NOT NULL,
|
||||||
|
"State" INTEGER NOT NULL,
|
||||||
|
"PublishAttempts" INTEGER NOT NULL,
|
||||||
|
"ProcessingAttempts" INTEGER NOT NULL,
|
||||||
|
"PublishedAt" TEXT NULL,
|
||||||
|
"ProcessingToken" TEXT NULL,
|
||||||
|
"LeaseExpiresAt" TEXT NULL,
|
||||||
|
"CompletedAt" TEXT NULL,
|
||||||
|
"LastError" TEXT NULL,
|
||||||
|
"CreatedAt" TEXT NOT NULL,
|
||||||
|
"UpdatedAt" TEXT NOT NULL
|
||||||
|
);
|
||||||
|
""",
|
||||||
|
"""
|
||||||
|
CREATE UNIQUE INDEX "IX_BackgroundJobOutboxMessages_JobKind_JobId"
|
||||||
|
ON "BackgroundJobOutboxMessages" ("JobKind", "JobId");
|
||||||
|
""",
|
||||||
|
"""
|
||||||
|
CREATE INDEX "IX_BackgroundJobOutboxMessages_LeaseExpiresAt"
|
||||||
|
ON "BackgroundJobOutboxMessages" ("LeaseExpiresAt");
|
||||||
|
""",
|
||||||
|
"""
|
||||||
|
CREATE INDEX "IX_BackgroundJobOutboxMessages_State_CreatedAt"
|
||||||
|
ON "BackgroundJobOutboxMessages" ("State", "CreatedAt");
|
||||||
|
"""
|
||||||
|
];
|
||||||
}
|
}
|
||||||
|
|||||||
+4845
File diff suppressed because it is too large
Load Diff
+62
@@ -0,0 +1,62 @@
|
|||||||
|
using System;
|
||||||
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
|
||||||
|
#nullable disable
|
||||||
|
|
||||||
|
namespace Jiaowu.Api.Infrastructure.Persistence.Migrations.MySql
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
public partial class BackgroundJobOutbox : Migration
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Up(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
migrationBuilder.CreateTable(
|
||||||
|
name: "BackgroundJobOutboxMessages",
|
||||||
|
columns: table => new
|
||||||
|
{
|
||||||
|
Id = table.Column<Guid>(type: "char(36)", nullable: false),
|
||||||
|
JobKind = table.Column<int>(type: "int", nullable: false),
|
||||||
|
JobId = table.Column<Guid>(type: "char(36)", nullable: false),
|
||||||
|
State = table.Column<int>(type: "int", nullable: false),
|
||||||
|
PublishAttempts = table.Column<int>(type: "int", nullable: false),
|
||||||
|
ProcessingAttempts = table.Column<int>(type: "int", nullable: false),
|
||||||
|
PublishedAt = table.Column<DateTime>(type: "datetime(6)", nullable: true),
|
||||||
|
ProcessingToken = table.Column<Guid>(type: "char(36)", nullable: true),
|
||||||
|
LeaseExpiresAt = table.Column<DateTime>(type: "datetime(6)", nullable: true),
|
||||||
|
CompletedAt = table.Column<DateTime>(type: "datetime(6)", nullable: true),
|
||||||
|
LastError = table.Column<string>(type: "varchar(2000)", maxLength: 2000, nullable: true),
|
||||||
|
CreatedAt = table.Column<DateTime>(type: "datetime(6)", nullable: false),
|
||||||
|
UpdatedAt = table.Column<DateTime>(type: "datetime(6)", nullable: false)
|
||||||
|
},
|
||||||
|
constraints: table =>
|
||||||
|
{
|
||||||
|
table.PrimaryKey("PK_BackgroundJobOutboxMessages", x => x.Id);
|
||||||
|
})
|
||||||
|
.Annotation("MySQL:Charset", "utf8mb4");
|
||||||
|
|
||||||
|
migrationBuilder.CreateIndex(
|
||||||
|
name: "IX_BackgroundJobOutboxMessages_JobKind_JobId",
|
||||||
|
table: "BackgroundJobOutboxMessages",
|
||||||
|
columns: new[] { "JobKind", "JobId" },
|
||||||
|
unique: true);
|
||||||
|
|
||||||
|
migrationBuilder.CreateIndex(
|
||||||
|
name: "IX_BackgroundJobOutboxMessages_LeaseExpiresAt",
|
||||||
|
table: "BackgroundJobOutboxMessages",
|
||||||
|
column: "LeaseExpiresAt");
|
||||||
|
|
||||||
|
migrationBuilder.CreateIndex(
|
||||||
|
name: "IX_BackgroundJobOutboxMessages_State_CreatedAt",
|
||||||
|
table: "BackgroundJobOutboxMessages",
|
||||||
|
columns: new[] { "State", "CreatedAt" });
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Down(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
migrationBuilder.DropTable(
|
||||||
|
name: "BackgroundJobOutboxMessages");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+55
@@ -3423,6 +3423,61 @@ namespace Jiaowu.Api.Infrastructure.Persistence.Migrations.MySql
|
|||||||
b.ToTable("AuditLogs");
|
b.ToTable("AuditLogs");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("Jiaowu.Api.Domain.System.BackgroundJobOutboxMessage", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("char(36)");
|
||||||
|
|
||||||
|
b.Property<DateTime?>("CompletedAt")
|
||||||
|
.HasColumnType("datetime(6)");
|
||||||
|
|
||||||
|
b.Property<DateTime>("CreatedAt")
|
||||||
|
.HasColumnType("datetime(6)");
|
||||||
|
|
||||||
|
b.Property<Guid>("JobId")
|
||||||
|
.HasColumnType("char(36)");
|
||||||
|
|
||||||
|
b.Property<int>("JobKind")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<string>("LastError")
|
||||||
|
.HasMaxLength(2000)
|
||||||
|
.HasColumnType("varchar(2000)");
|
||||||
|
|
||||||
|
b.Property<DateTime?>("LeaseExpiresAt")
|
||||||
|
.HasColumnType("datetime(6)");
|
||||||
|
|
||||||
|
b.Property<int>("ProcessingAttempts")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<Guid?>("ProcessingToken")
|
||||||
|
.HasColumnType("char(36)");
|
||||||
|
|
||||||
|
b.Property<int>("PublishAttempts")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<DateTime?>("PublishedAt")
|
||||||
|
.HasColumnType("datetime(6)");
|
||||||
|
|
||||||
|
b.Property<int>("State")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<DateTime>("UpdatedAt")
|
||||||
|
.HasColumnType("datetime(6)");
|
||||||
|
|
||||||
|
b.HasKey("Id");
|
||||||
|
|
||||||
|
b.HasIndex("LeaseExpiresAt");
|
||||||
|
|
||||||
|
b.HasIndex("JobKind", "JobId")
|
||||||
|
.IsUnique();
|
||||||
|
|
||||||
|
b.HasIndex("State", "CreatedAt");
|
||||||
|
|
||||||
|
b.ToTable("BackgroundJobOutboxMessages");
|
||||||
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<System.Guid>", b =>
|
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<System.Guid>", b =>
|
||||||
{
|
{
|
||||||
b.Property<int>("Id")
|
b.Property<int>("Id")
|
||||||
|
|||||||
@@ -1,104 +1,11 @@
|
|||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using System.Threading.Channels;
|
|
||||||
using Jiaowu.Api.Domain.Academic;
|
using Jiaowu.Api.Domain.Academic;
|
||||||
using Jiaowu.Api.Infrastructure.Persistence;
|
using Jiaowu.Api.Infrastructure.Persistence;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
|
||||||
namespace Jiaowu.Api.Infrastructure.Scheduling;
|
namespace Jiaowu.Api.Infrastructure.Scheduling;
|
||||||
|
|
||||||
public sealed class AutomaticScheduleJobQueue
|
|
||||||
{
|
|
||||||
private readonly Channel<Guid> _channel = Channel.CreateUnbounded<Guid>(
|
|
||||||
new UnboundedChannelOptions
|
|
||||||
{
|
|
||||||
SingleReader = true,
|
|
||||||
SingleWriter = false
|
|
||||||
});
|
|
||||||
|
|
||||||
public void Enqueue(Guid jobId)
|
|
||||||
{
|
|
||||||
if (!_channel.Writer.TryWrite(jobId))
|
|
||||||
throw new InvalidOperationException("自动排课任务队列当前不可用。");
|
|
||||||
}
|
|
||||||
|
|
||||||
public IAsyncEnumerable<Guid> ReadAllAsync(CancellationToken cancellationToken) =>
|
|
||||||
_channel.Reader.ReadAllAsync(cancellationToken);
|
|
||||||
}
|
|
||||||
|
|
||||||
public sealed class AutomaticScheduleJobWorker(
|
|
||||||
IServiceScopeFactory scopeFactory,
|
|
||||||
AutomaticScheduleJobQueue queue,
|
|
||||||
ILogger<AutomaticScheduleJobWorker> logger) : BackgroundService
|
|
||||||
{
|
|
||||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
|
||||||
{
|
|
||||||
await RecoverInterruptedJobsAsync(stoppingToken);
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
await foreach (var jobId in queue.ReadAllAsync(stoppingToken))
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
await using var scope = scopeFactory.CreateAsyncScope();
|
|
||||||
var processor = scope.ServiceProvider
|
|
||||||
.GetRequiredService<AutomaticScheduleJobProcessor>();
|
|
||||||
await processor.ProcessAsync(jobId, stoppingToken);
|
|
||||||
}
|
|
||||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
|
||||||
{
|
|
||||||
throw;
|
|
||||||
}
|
|
||||||
catch (Exception exception)
|
|
||||||
{
|
|
||||||
logger.LogError(
|
|
||||||
exception,
|
|
||||||
"Unexpected failure while dispatching automatic schedule job {JobId}.",
|
|
||||||
jobId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
|
||||||
{
|
|
||||||
logger.LogInformation("Automatic schedule job worker is stopping.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async Task RecoverInterruptedJobsAsync(CancellationToken cancellationToken)
|
|
||||||
{
|
|
||||||
await using var scope = scopeFactory.CreateAsyncScope();
|
|
||||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
|
||||||
var jobs = await db.AutomaticScheduleJobs
|
|
||||||
.Where(x =>
|
|
||||||
x.Status == AutomaticScheduleJobStatus.Queued ||
|
|
||||||
x.Status == AutomaticScheduleJobStatus.Running)
|
|
||||||
.OrderBy(x => x.CreatedAt)
|
|
||||||
.ToListAsync(cancellationToken);
|
|
||||||
|
|
||||||
foreach (var job in jobs)
|
|
||||||
{
|
|
||||||
job.Status = AutomaticScheduleJobStatus.Queued;
|
|
||||||
job.ActiveSchedulePlanId = job.SchedulePlanId;
|
|
||||||
job.StartedAt = null;
|
|
||||||
job.CompletedAt = null;
|
|
||||||
job.ErrorMessage = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (jobs.Count > 0)
|
|
||||||
await db.SaveChangesAsync(cancellationToken);
|
|
||||||
foreach (var job in jobs)
|
|
||||||
queue.Enqueue(job.Id);
|
|
||||||
|
|
||||||
if (jobs.Count > 0)
|
|
||||||
{
|
|
||||||
logger.LogInformation(
|
|
||||||
"Recovered {JobCount} queued or interrupted automatic schedule jobs.",
|
|
||||||
jobs.Count);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public sealed class AutomaticScheduleJobProcessor(
|
public sealed class AutomaticScheduleJobProcessor(
|
||||||
AppDbContext db,
|
AppDbContext db,
|
||||||
AutomaticScheduleGenerator generator,
|
AutomaticScheduleGenerator generator,
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
using System.Threading.Channels;
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
using System.Diagnostics.CodeAnalysis;
|
||||||
using Jiaowu.Api.Domain.Academic;
|
using Jiaowu.Api.Domain.Academic;
|
||||||
using Jiaowu.Api.Infrastructure.Caching;
|
using Jiaowu.Api.Infrastructure.Caching;
|
||||||
@@ -7,100 +6,6 @@ using Microsoft.EntityFrameworkCore;
|
|||||||
|
|
||||||
namespace Jiaowu.Api.Infrastructure.Scheduling;
|
namespace Jiaowu.Api.Infrastructure.Scheduling;
|
||||||
|
|
||||||
public sealed class SchedulePublishJobQueue
|
|
||||||
{
|
|
||||||
private readonly Channel<Guid> _channel = Channel.CreateUnbounded<Guid>(
|
|
||||||
new UnboundedChannelOptions
|
|
||||||
{
|
|
||||||
SingleReader = true,
|
|
||||||
SingleWriter = false
|
|
||||||
});
|
|
||||||
|
|
||||||
public void Enqueue(Guid jobId)
|
|
||||||
{
|
|
||||||
if (!_channel.Writer.TryWrite(jobId))
|
|
||||||
throw new InvalidOperationException("课表发布任务队列当前不可用。");
|
|
||||||
}
|
|
||||||
|
|
||||||
public IAsyncEnumerable<Guid> ReadAllAsync(CancellationToken cancellationToken) =>
|
|
||||||
_channel.Reader.ReadAllAsync(cancellationToken);
|
|
||||||
}
|
|
||||||
|
|
||||||
public sealed class SchedulePublishJobWorker(
|
|
||||||
IServiceScopeFactory scopeFactory,
|
|
||||||
SchedulePublishJobQueue queue,
|
|
||||||
ILogger<SchedulePublishJobWorker> logger) : BackgroundService
|
|
||||||
{
|
|
||||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
|
||||||
{
|
|
||||||
await RecoverInterruptedJobsAsync(stoppingToken);
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
await foreach (var jobId in queue.ReadAllAsync(stoppingToken))
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
await using var scope = scopeFactory.CreateAsyncScope();
|
|
||||||
var processor = scope.ServiceProvider
|
|
||||||
.GetRequiredService<SchedulePublishJobProcessor>();
|
|
||||||
await processor.ProcessAsync(jobId, stoppingToken);
|
|
||||||
}
|
|
||||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
|
||||||
{
|
|
||||||
throw;
|
|
||||||
}
|
|
||||||
catch (Exception exception)
|
|
||||||
{
|
|
||||||
logger.LogError(
|
|
||||||
exception,
|
|
||||||
"Unexpected failure while dispatching schedule publish job {JobId}.",
|
|
||||||
jobId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
|
||||||
{
|
|
||||||
logger.LogInformation("Schedule publish job worker is stopping.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private async Task RecoverInterruptedJobsAsync(CancellationToken cancellationToken)
|
|
||||||
{
|
|
||||||
await using var scope = scopeFactory.CreateAsyncScope();
|
|
||||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
|
||||||
var jobs = await db.SchedulePublishJobs
|
|
||||||
.Where(x =>
|
|
||||||
x.Status == SchedulePublishJobStatus.Queued ||
|
|
||||||
x.Status == SchedulePublishJobStatus.Running)
|
|
||||||
.OrderBy(x => x.CreatedAt)
|
|
||||||
.ToListAsync(cancellationToken);
|
|
||||||
|
|
||||||
foreach (var job in jobs)
|
|
||||||
{
|
|
||||||
job.Status = SchedulePublishJobStatus.Queued;
|
|
||||||
job.ActiveAcademicTermId = job.AcademicTermId;
|
|
||||||
job.CompletedSteps = 0;
|
|
||||||
job.CurrentStep = "等待后台检查";
|
|
||||||
job.StartedAt = null;
|
|
||||||
job.CompletedAt = null;
|
|
||||||
job.ErrorMessage = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (jobs.Count > 0)
|
|
||||||
await db.SaveChangesAsync(cancellationToken);
|
|
||||||
foreach (var job in jobs)
|
|
||||||
queue.Enqueue(job.Id);
|
|
||||||
|
|
||||||
if (jobs.Count > 0)
|
|
||||||
{
|
|
||||||
logger.LogInformation(
|
|
||||||
"Recovered {JobCount} queued or interrupted schedule publish jobs.",
|
|
||||||
jobs.Count);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public sealed class SchedulePublishJobProcessor(
|
public sealed class SchedulePublishJobProcessor(
|
||||||
AppDbContext db,
|
AppDbContext db,
|
||||||
SchedulePlanPublisher publisher,
|
SchedulePlanPublisher publisher,
|
||||||
|
|||||||
@@ -32,6 +32,7 @@
|
|||||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.10" />
|
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.10" />
|
||||||
<PackageReference Include="MySql.EntityFrameworkCore" Version="10.0.7" />
|
<PackageReference Include="MySql.EntityFrameworkCore" Version="10.0.7" />
|
||||||
<PackageReference Include="QRCoder" Version="1.8.0" />
|
<PackageReference Include="QRCoder" Version="1.8.0" />
|
||||||
|
<PackageReference Include="RabbitMQ.Client" Version="7.2.1" />
|
||||||
<PackageReference Include="SkiaSharp" Version="3.119.2" />
|
<PackageReference Include="SkiaSharp" Version="3.119.2" />
|
||||||
<PackageReference Include="SkiaSharp.NativeAssets.Linux.NoDependencies" Version="3.119.2" />
|
<PackageReference Include="SkiaSharp.NativeAssets.Linux.NoDependencies" Version="3.119.2" />
|
||||||
<PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.4" />
|
<PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.4" />
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
using System.Text;
|
using System.Text;
|
||||||
using System.Text.Json.Serialization;
|
using System.Text.Json.Serialization;
|
||||||
using Jiaowu.Api.Domain.Identity;
|
using Jiaowu.Api.Domain.Identity;
|
||||||
|
using Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||||
using Jiaowu.Api.Infrastructure.Configuration;
|
using Jiaowu.Api.Infrastructure.Configuration;
|
||||||
using Jiaowu.Api.Infrastructure.Auth;
|
using Jiaowu.Api.Infrastructure.Auth;
|
||||||
using Jiaowu.Api.Infrastructure.Caching;
|
using Jiaowu.Api.Infrastructure.Caching;
|
||||||
@@ -70,6 +71,12 @@ var cacheOptions = builder.Configuration
|
|||||||
var officialDocumentOptions = builder.Configuration
|
var officialDocumentOptions = builder.Configuration
|
||||||
.GetSection(OfficialDocumentOptions.SectionName)
|
.GetSection(OfficialDocumentOptions.SectionName)
|
||||||
.Get<OfficialDocumentOptions>() ?? new OfficialDocumentOptions();
|
.Get<OfficialDocumentOptions>() ?? new OfficialDocumentOptions();
|
||||||
|
var backgroundJobOptions = builder.Configuration
|
||||||
|
.GetSection(BackgroundJobOptions.SectionName)
|
||||||
|
.Get<BackgroundJobOptions>() ?? new BackgroundJobOptions();
|
||||||
|
var rabbitMqOptions = builder.Configuration
|
||||||
|
.GetSection(RabbitMqOptions.SectionName)
|
||||||
|
.Get<RabbitMqOptions>() ?? new RabbitMqOptions();
|
||||||
|
|
||||||
if (string.IsNullOrWhiteSpace(officialDocumentOptions.InstitutionName) ||
|
if (string.IsNullOrWhiteSpace(officialDocumentOptions.InstitutionName) ||
|
||||||
string.IsNullOrWhiteSpace(officialDocumentOptions.IssuingOffice) ||
|
string.IsNullOrWhiteSpace(officialDocumentOptions.IssuingOffice) ||
|
||||||
@@ -119,9 +126,46 @@ if (cacheOptions.ReferenceExpirationMinutes is < 1 or > 1440 ||
|
|||||||
"Cache 缓存时间或 MaximumPayloadKilobytes 超出允许范围。");
|
"Cache 缓存时间或 MaximumPayloadKilobytes 超出允许范围。");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!backgroundJobOptions.Transport.Equals(
|
||||||
|
"InMemory",
|
||||||
|
StringComparison.OrdinalIgnoreCase) &&
|
||||||
|
!backgroundJobOptions.UsesRabbitMq)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
"BackgroundJobs:Transport 仅支持 InMemory 或 RabbitMq。");
|
||||||
|
}
|
||||||
|
if (backgroundJobOptions.PollIntervalMilliseconds is < 100 or > 30000 ||
|
||||||
|
backgroundJobOptions.LeaseSeconds is < 30 or > 3600 ||
|
||||||
|
backgroundJobOptions.PrefetchCount is < 1 or > 100 ||
|
||||||
|
backgroundJobOptions.ProcessingAttemptLimit is < 1 or > 100 ||
|
||||||
|
string.IsNullOrWhiteSpace(backgroundJobOptions.Exchange) ||
|
||||||
|
string.IsNullOrWhiteSpace(backgroundJobOptions.QueuePrefix))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("BackgroundJobs 配置超出允许范围。");
|
||||||
|
}
|
||||||
|
if (backgroundJobOptions.UsesRabbitMq &&
|
||||||
|
(string.IsNullOrWhiteSpace(rabbitMqOptions.HostName) ||
|
||||||
|
rabbitMqOptions.Port is < 1 or > 65535 ||
|
||||||
|
string.IsNullOrWhiteSpace(rabbitMqOptions.UserName) ||
|
||||||
|
string.IsNullOrWhiteSpace(rabbitMqOptions.Password) ||
|
||||||
|
string.IsNullOrWhiteSpace(rabbitMqOptions.VirtualHost)))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("RabbitMq 连接配置不完整。");
|
||||||
|
}
|
||||||
|
if (backgroundJobOptions.UsesRabbitMq &&
|
||||||
|
!builder.Environment.IsDevelopment() &&
|
||||||
|
(rabbitMqOptions.UserName.Equals("guest", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
rabbitMqOptions.Password == "guest"))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
"生产环境启用 RabbitMQ 时不能使用默认 guest 凭据。");
|
||||||
|
}
|
||||||
|
|
||||||
builder.Services.AddSingleton(databaseOptions);
|
builder.Services.AddSingleton(databaseOptions);
|
||||||
builder.Services.AddSingleton(cacheOptions);
|
builder.Services.AddSingleton(cacheOptions);
|
||||||
builder.Services.AddSingleton(officialDocumentOptions);
|
builder.Services.AddSingleton(officialDocumentOptions);
|
||||||
|
builder.Services.AddSingleton(backgroundJobOptions);
|
||||||
|
builder.Services.AddSingleton(rabbitMqOptions);
|
||||||
builder.Services.Configure<OfficialDocumentOptions>(
|
builder.Services.Configure<OfficialDocumentOptions>(
|
||||||
builder.Configuration.GetSection(OfficialDocumentOptions.SectionName));
|
builder.Configuration.GetSection(OfficialDocumentOptions.SectionName));
|
||||||
builder.Services.AddDbContextPool<AppDbContext>(options =>
|
builder.Services.AddDbContextPool<AppDbContext>(options =>
|
||||||
@@ -218,19 +262,29 @@ builder.Services.AddScoped<AutomaticScheduleGenerator>();
|
|||||||
builder.Services.AddScoped<PersonalCalendarService>();
|
builder.Services.AddScoped<PersonalCalendarService>();
|
||||||
builder.Services.AddScoped<ClassroomReservationAvailabilityService>();
|
builder.Services.AddScoped<ClassroomReservationAvailabilityService>();
|
||||||
builder.Services.AddScoped<AutomaticScheduleJobProcessor>();
|
builder.Services.AddScoped<AutomaticScheduleJobProcessor>();
|
||||||
builder.Services.AddSingleton<AutomaticScheduleJobQueue>();
|
|
||||||
builder.Services.AddHostedService<AutomaticScheduleJobWorker>();
|
|
||||||
builder.Services.AddScoped<SchedulePlanPublisher>();
|
builder.Services.AddScoped<SchedulePlanPublisher>();
|
||||||
builder.Services.AddScoped<SchedulePublishJobProcessor>();
|
builder.Services.AddScoped<SchedulePublishJobProcessor>();
|
||||||
builder.Services.AddSingleton<SchedulePublishJobQueue>();
|
|
||||||
builder.Services.AddHostedService<SchedulePublishJobWorker>();
|
|
||||||
builder.Services.AddHostedService<WarningCheckWorker>();
|
builder.Services.AddHostedService<WarningCheckWorker>();
|
||||||
builder.Services.AddScoped<ExamArrangementService>();
|
builder.Services.AddScoped<ExamArrangementService>();
|
||||||
builder.Services.AddScoped<MakeupExamEligibilityService>();
|
builder.Services.AddScoped<MakeupExamEligibilityService>();
|
||||||
builder.Services.AddScoped<MakeupExamArrangementService>();
|
builder.Services.AddScoped<MakeupExamArrangementService>();
|
||||||
builder.Services.AddSingleton<MakeupExamAutoJobQueue>();
|
|
||||||
builder.Services.AddHostedService<MakeupExamAutoJobWorker>();
|
|
||||||
builder.Services.AddScoped<MakeupExamAutoJobProcessor>();
|
builder.Services.AddScoped<MakeupExamAutoJobProcessor>();
|
||||||
|
builder.Services.AddSingleton<BackgroundJobRunner>();
|
||||||
|
if (backgroundJobOptions.UsesRabbitMq)
|
||||||
|
{
|
||||||
|
builder.Services.AddSingleton<
|
||||||
|
IBackgroundJobTransport,
|
||||||
|
RabbitMqBackgroundJobTransport>();
|
||||||
|
builder.Services.AddHostedService<RabbitMqBackgroundJobWorker>();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
builder.Services.AddSingleton<InMemoryBackgroundJobTransport>();
|
||||||
|
builder.Services.AddSingleton<IBackgroundJobTransport>(services =>
|
||||||
|
services.GetRequiredService<InMemoryBackgroundJobTransport>());
|
||||||
|
builder.Services.AddHostedService<InMemoryBackgroundJobWorker>();
|
||||||
|
}
|
||||||
|
builder.Services.AddHostedService<BackgroundJobOutboxPublisher>();
|
||||||
builder.Services.AddSingleton<IOfficialDocumentPdfGenerator, OfficialDocumentPdfGenerator>();
|
builder.Services.AddSingleton<IOfficialDocumentPdfGenerator, OfficialDocumentPdfGenerator>();
|
||||||
builder.Services.AddScoped<OfficialDocumentService>();
|
builder.Services.AddScoped<OfficialDocumentService>();
|
||||||
|
|
||||||
@@ -391,6 +445,7 @@ app.MapGet("/health/live", () => Results.Ok(new { Status = "healthy" }))
|
|||||||
app.MapGet("/health", CheckDatabaseHealthAsync).AllowAnonymous();
|
app.MapGet("/health", CheckDatabaseHealthAsync).AllowAnonymous();
|
||||||
app.MapGet("/health/ready", CheckDatabaseHealthAsync).AllowAnonymous();
|
app.MapGet("/health/ready", CheckDatabaseHealthAsync).AllowAnonymous();
|
||||||
app.MapGet("/health/cache", CheckCacheHealthAsync).AllowAnonymous();
|
app.MapGet("/health/cache", CheckCacheHealthAsync).AllowAnonymous();
|
||||||
|
app.MapGet("/health/messaging", CheckMessagingHealthAsync).AllowAnonymous();
|
||||||
app.MapFallback(async context =>
|
app.MapFallback(async context =>
|
||||||
{
|
{
|
||||||
if (context.Request.Path.StartsWithSegments("/api") ||
|
if (context.Request.Path.StartsWithSegments("/api") ||
|
||||||
@@ -477,4 +532,17 @@ static async Task<IResult> CheckCacheHealthAsync(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static async Task<IResult> CheckMessagingHealthAsync(
|
||||||
|
IBackgroundJobTransport transport,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var healthy = await transport.CheckHealthAsync(cancellationToken);
|
||||||
|
var backend = transport.IsDurable ? "rabbitmq" : "memory";
|
||||||
|
return healthy
|
||||||
|
? Results.Ok(new { Status = "healthy", Backend = backend })
|
||||||
|
: Results.Json(
|
||||||
|
new { Status = "unhealthy", Backend = backend },
|
||||||
|
statusCode: StatusCodes.Status503ServiceUnavailable);
|
||||||
|
}
|
||||||
|
|
||||||
public partial class Program;
|
public partial class Program;
|
||||||
|
|||||||
@@ -19,6 +19,25 @@
|
|||||||
"AnalyticsLocalExpirationSeconds": 30,
|
"AnalyticsLocalExpirationSeconds": 30,
|
||||||
"MaximumPayloadKilobytes": 2048
|
"MaximumPayloadKilobytes": 2048
|
||||||
},
|
},
|
||||||
|
"BackgroundJobs": {
|
||||||
|
"Transport": "InMemory",
|
||||||
|
"PollIntervalMilliseconds": 500,
|
||||||
|
"LeaseSeconds": 120,
|
||||||
|
"PrefetchCount": 1,
|
||||||
|
"Exchange": "jiaowu.background-jobs",
|
||||||
|
"QueuePrefix": "jiaowu.background-jobs",
|
||||||
|
"UseQuorumQueues": true,
|
||||||
|
"ProcessingAttemptLimit": 5
|
||||||
|
},
|
||||||
|
"RabbitMq": {
|
||||||
|
"HostName": "localhost",
|
||||||
|
"Port": 5672,
|
||||||
|
"UserName": "guest",
|
||||||
|
"Password": "guest",
|
||||||
|
"VirtualHost": "/",
|
||||||
|
"UseTls": false,
|
||||||
|
"TlsServerName": ""
|
||||||
|
},
|
||||||
"Jwt": {
|
"Jwt": {
|
||||||
"Issuer": "Jiaowu.Api",
|
"Issuer": "Jiaowu.Api",
|
||||||
"Audience": "Jiaowu.Web",
|
"Audience": "Jiaowu.Web",
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ public sealed class AutomaticScheduleGeneratorTests
|
|||||||
.Options;
|
.Options;
|
||||||
await using var db = new AppDbContext(options);
|
await using var db = new AppDbContext(options);
|
||||||
await db.Database.EnsureCreatedAsync();
|
await db.Database.EnsureCreatedAsync();
|
||||||
|
await db.Database.ExecuteSqlRawAsync(
|
||||||
|
"DROP TABLE \"BackgroundJobOutboxMessages\"");
|
||||||
|
|
||||||
var migrator = new DevelopmentSqliteMigrator(
|
var migrator = new DevelopmentSqliteMigrator(
|
||||||
db,
|
db,
|
||||||
@@ -27,6 +29,7 @@ public sealed class AutomaticScheduleGeneratorTests
|
|||||||
|
|
||||||
Assert.True(await db.ScheduleTimeSlots.CountAsync() == 0);
|
Assert.True(await db.ScheduleTimeSlots.CountAsync() == 0);
|
||||||
Assert.True(await db.AutomaticScheduleJobs.CountAsync() == 0);
|
Assert.True(await db.AutomaticScheduleJobs.CountAsync() == 0);
|
||||||
|
Assert.True(await db.BackgroundJobOutboxMessages.CountAsync() == 0);
|
||||||
Assert.True(await db.Database
|
Assert.True(await db.Database
|
||||||
.SqlQueryRaw<int>(
|
.SqlQueryRaw<int>(
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -0,0 +1,182 @@
|
|||||||
|
using Jiaowu.Api.Domain.Academic;
|
||||||
|
using Jiaowu.Api.Domain.System;
|
||||||
|
using Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||||
|
using Jiaowu.Api.Infrastructure.Persistence;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
|
||||||
|
namespace Jiaowu.Api.Tests;
|
||||||
|
|
||||||
|
public sealed class BackgroundJobOutboxTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task Publisher_moves_pending_message_to_in_memory_transport()
|
||||||
|
{
|
||||||
|
var databasePath = Path.Combine(
|
||||||
|
Path.GetTempPath(),
|
||||||
|
$"jiaowu-outbox-{Guid.NewGuid():N}.sqlite");
|
||||||
|
var services = new ServiceCollection();
|
||||||
|
services.AddLogging();
|
||||||
|
services.AddDbContext<AppDbContext>(options =>
|
||||||
|
options.UseSqlite($"Data Source={databasePath};Pooling=False"));
|
||||||
|
var options = new BackgroundJobOptions
|
||||||
|
{
|
||||||
|
PollIntervalMilliseconds = 100
|
||||||
|
};
|
||||||
|
services.AddSingleton(options);
|
||||||
|
services.AddSingleton<InMemoryBackgroundJobTransport>();
|
||||||
|
services.AddSingleton<IBackgroundJobTransport>(provider =>
|
||||||
|
provider.GetRequiredService<InMemoryBackgroundJobTransport>());
|
||||||
|
services.AddSingleton<BackgroundJobOutboxPublisher>();
|
||||||
|
var provider = services.BuildServiceProvider();
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Guid jobId;
|
||||||
|
await using (var scope = provider.CreateAsyncScope())
|
||||||
|
{
|
||||||
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||||
|
await db.Database.EnsureCreatedAsync();
|
||||||
|
var term = new AcademicTerm
|
||||||
|
{
|
||||||
|
Code = "RECOVERY",
|
||||||
|
Name = "补投测试学期",
|
||||||
|
AcademicYear = "2026-2027",
|
||||||
|
Season = TermSeason.Autumn,
|
||||||
|
StartDate = new DateOnly(2026, 9, 1),
|
||||||
|
EndDate = new DateOnly(2027, 1, 15)
|
||||||
|
};
|
||||||
|
var plan = new SchedulePlan
|
||||||
|
{
|
||||||
|
AcademicTerm = term,
|
||||||
|
Name = "补投测试排课",
|
||||||
|
Version = "V1"
|
||||||
|
};
|
||||||
|
var job = new AutomaticScheduleJob
|
||||||
|
{
|
||||||
|
SchedulePlan = plan,
|
||||||
|
ActiveSchedulePlanId = plan.Id
|
||||||
|
};
|
||||||
|
jobId = job.Id;
|
||||||
|
db.AddRange(term, plan, job);
|
||||||
|
await db.SaveChangesAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
var publisher = provider.GetRequiredService<BackgroundJobOutboxPublisher>();
|
||||||
|
await publisher.StartAsync(CancellationToken.None);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var published = false;
|
||||||
|
for (var attempt = 0; attempt < 50 && !published; attempt++)
|
||||||
|
{
|
||||||
|
await Task.Delay(50);
|
||||||
|
await using var scope = provider.CreateAsyncScope();
|
||||||
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||||
|
published = await db.BackgroundJobOutboxMessages.AsNoTracking()
|
||||||
|
.AnyAsync(x =>
|
||||||
|
x.JobId == jobId &&
|
||||||
|
x.JobKind == BackgroundJobKind.AutomaticSchedule &&
|
||||||
|
x.State == BackgroundJobOutboxState.Published);
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.True(published);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
await publisher.StopAsync(CancellationToken.None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
await provider.DisposeAsync();
|
||||||
|
File.Delete(databasePath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Runner_stops_after_retry_limit_and_ignores_duplicate_delivery()
|
||||||
|
{
|
||||||
|
var databasePath = Path.Combine(
|
||||||
|
Path.GetTempPath(),
|
||||||
|
$"jiaowu-runner-{Guid.NewGuid():N}.sqlite");
|
||||||
|
var services = new ServiceCollection();
|
||||||
|
services.AddLogging();
|
||||||
|
services.AddDbContext<AppDbContext>(options =>
|
||||||
|
options.UseSqlite($"Data Source={databasePath};Pooling=False"));
|
||||||
|
services.AddScoped<Jiaowu.Api.Infrastructure.Scheduling.AutomaticScheduleGenerator>();
|
||||||
|
services.AddScoped<Jiaowu.Api.Infrastructure.Scheduling.AutomaticScheduleJobProcessor>();
|
||||||
|
services.AddSingleton(new BackgroundJobOptions());
|
||||||
|
services.AddSingleton<InMemoryBackgroundJobTransport>();
|
||||||
|
services.AddSingleton<IBackgroundJobTransport>(provider =>
|
||||||
|
provider.GetRequiredService<InMemoryBackgroundJobTransport>());
|
||||||
|
services.AddSingleton<BackgroundJobRunner>();
|
||||||
|
var provider = services.BuildServiceProvider();
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
BackgroundJobEnvelope envelope;
|
||||||
|
await using (var scope = provider.CreateAsyncScope())
|
||||||
|
{
|
||||||
|
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||||
|
await db.Database.EnsureCreatedAsync();
|
||||||
|
var term = new AcademicTerm
|
||||||
|
{
|
||||||
|
Code = "OUTBOX",
|
||||||
|
Name = "消息测试学期",
|
||||||
|
AcademicYear = "2026-2027",
|
||||||
|
Season = TermSeason.Autumn,
|
||||||
|
StartDate = new DateOnly(2026, 9, 1),
|
||||||
|
EndDate = new DateOnly(2027, 1, 15)
|
||||||
|
};
|
||||||
|
var plan = new SchedulePlan
|
||||||
|
{
|
||||||
|
AcademicTerm = term,
|
||||||
|
Name = "消息测试排课",
|
||||||
|
Version = "V1"
|
||||||
|
};
|
||||||
|
var job = new AutomaticScheduleJob
|
||||||
|
{
|
||||||
|
SchedulePlan = plan,
|
||||||
|
ActiveSchedulePlanId = plan.Id,
|
||||||
|
Status = AutomaticScheduleJobStatus.Queued
|
||||||
|
};
|
||||||
|
var outbox = BackgroundJobOutboxMessage.Create(
|
||||||
|
BackgroundJobKind.AutomaticSchedule,
|
||||||
|
job.Id);
|
||||||
|
outbox.State = BackgroundJobOutboxState.Published;
|
||||||
|
outbox.ProcessingAttempts = 5;
|
||||||
|
db.AddRange(term, plan, job, outbox);
|
||||||
|
await db.SaveChangesAsync();
|
||||||
|
envelope = new BackgroundJobEnvelope(
|
||||||
|
outbox.Id,
|
||||||
|
outbox.JobKind,
|
||||||
|
outbox.JobId);
|
||||||
|
}
|
||||||
|
|
||||||
|
var runner = provider.GetRequiredService<BackgroundJobRunner>();
|
||||||
|
var first = await runner.RunAsync(envelope, CancellationToken.None);
|
||||||
|
var duplicate = await runner.RunAsync(envelope, CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Equal(BackgroundJobRunOutcome.Completed, first.Outcome);
|
||||||
|
Assert.Equal(BackgroundJobRunOutcome.Completed, duplicate.Outcome);
|
||||||
|
await using var assertScope = provider.CreateAsyncScope();
|
||||||
|
var assertDb = assertScope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||||
|
var persistedOutbox =
|
||||||
|
await assertDb.BackgroundJobOutboxMessages.SingleAsync();
|
||||||
|
Assert.Equal(BackgroundJobOutboxState.Completed, persistedOutbox.State);
|
||||||
|
Assert.NotNull(persistedOutbox.CompletedAt);
|
||||||
|
Assert.Null(persistedOutbox.ProcessingToken);
|
||||||
|
Assert.Null(persistedOutbox.LeaseExpiresAt);
|
||||||
|
Assert.Equal(6, persistedOutbox.ProcessingAttempts);
|
||||||
|
var persistedJob = await assertDb.AutomaticScheduleJobs.SingleAsync();
|
||||||
|
Assert.Equal(AutomaticScheduleJobStatus.Failed, persistedJob.Status);
|
||||||
|
Assert.Null(persistedJob.ActiveSchedulePlanId);
|
||||||
|
Assert.Contains("超过 5 次", persistedJob.ErrorMessage);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
await provider.DisposeAsync();
|
||||||
|
File.Delete(databasePath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -301,7 +301,6 @@ public sealed class ScheduleSettingsControllerTests
|
|||||||
db,
|
db,
|
||||||
null!,
|
null!,
|
||||||
null!,
|
null!,
|
||||||
null!,
|
|
||||||
null!)
|
null!)
|
||||||
.GetTimeSlotsForTerm(term.Id, CancellationToken.None);
|
.GetTimeSlotsForTerm(term.Id, CancellationToken.None);
|
||||||
var makeupTimeSlotsOk =
|
var makeupTimeSlotsOk =
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
using Jiaowu.Api.Controllers;
|
using Jiaowu.Api.Controllers;
|
||||||
using Jiaowu.Api.Domain.Academic;
|
using Jiaowu.Api.Domain.Academic;
|
||||||
|
using Jiaowu.Api.Domain.System;
|
||||||
using Jiaowu.Api.Infrastructure.Persistence;
|
using Jiaowu.Api.Infrastructure.Persistence;
|
||||||
using Jiaowu.Api.Infrastructure.Scheduling;
|
using Jiaowu.Api.Infrastructure.Scheduling;
|
||||||
using Microsoft.AspNetCore.Http;
|
using Microsoft.AspNetCore.Http;
|
||||||
@@ -70,10 +71,7 @@ public sealed class SchedulesControllerTests
|
|||||||
db.AddRange(term, college, course, task, plan);
|
db.AddRange(term, college, course, task, plan);
|
||||||
await db.SaveChangesAsync();
|
await db.SaveChangesAsync();
|
||||||
|
|
||||||
var controller = new SchedulesController(
|
var controller = new SchedulesController(db)
|
||||||
db,
|
|
||||||
new AutomaticScheduleJobQueue(),
|
|
||||||
new SchedulePublishJobQueue())
|
|
||||||
{
|
{
|
||||||
ControllerContext = new ControllerContext
|
ControllerContext = new ControllerContext
|
||||||
{
|
{
|
||||||
@@ -90,6 +88,10 @@ public sealed class SchedulesControllerTests
|
|||||||
SchedulePlanStatus.Draft,
|
SchedulePlanStatus.Draft,
|
||||||
(await db.SchedulePlans.SingleAsync()).Status);
|
(await db.SchedulePlans.SingleAsync()).Status);
|
||||||
Assert.Equal(1, await db.SchedulePublishJobs.CountAsync());
|
Assert.Equal(1, await db.SchedulePublishJobs.CountAsync());
|
||||||
|
var outbox = await db.BackgroundJobOutboxMessages.SingleAsync();
|
||||||
|
Assert.Equal(BackgroundJobKind.SchedulePublish, outbox.JobKind);
|
||||||
|
Assert.Equal(response.Id, outbox.JobId);
|
||||||
|
Assert.Equal(BackgroundJobOutboxState.Pending, outbox.State);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -132,10 +134,7 @@ public sealed class SchedulesControllerTests
|
|||||||
db.AddRange(term, plan, completed);
|
db.AddRange(term, plan, completed);
|
||||||
await db.SaveChangesAsync();
|
await db.SaveChangesAsync();
|
||||||
|
|
||||||
var controller = new SchedulesController(
|
var controller = new SchedulesController(db);
|
||||||
db,
|
|
||||||
new AutomaticScheduleJobQueue(),
|
|
||||||
new SchedulePublishJobQueue());
|
|
||||||
var result = await controller.GetLatestAutomaticScheduleJob(
|
var result = await controller.GetLatestAutomaticScheduleJob(
|
||||||
plan.Id,
|
plan.Id,
|
||||||
CancellationToken.None);
|
CancellationToken.None);
|
||||||
|
|||||||
Reference in New Issue
Block a user