clickhouse
This commit is contained in:
@@ -8,6 +8,9 @@ MYSQL_USER=jiaowu
|
||||
MYSQL_PASSWORD=
|
||||
MYSQL_ROOT_PASSWORD=
|
||||
|
||||
CLICKHOUSE_USER=jiaowu_analytics
|
||||
CLICKHOUSE_PASSWORD=
|
||||
|
||||
RABBITMQ_USER=jiaowu
|
||||
RABBITMQ_PASSWORD=
|
||||
BACKGROUND_JOB_AUTOMATIC_SCHEDULE_CONCURRENCY=1
|
||||
|
||||
@@ -55,6 +55,18 @@ PerformanceReporting__Enabled=false
|
||||
PerformanceReporting__CacheSeconds=30
|
||||
PerformanceReporting__TimeoutSeconds=10
|
||||
|
||||
# ClickHouse 仅作为异步分析读模型,不参与教务事务写入。启用前请为应用创建
|
||||
# 仅能操作该分析库的独立账号,并通过 TLS 或受信任的内网访问。
|
||||
ClickHouseAnalytics__Enabled=false
|
||||
# ClickHouseAnalytics__Endpoint=https://clickhouse.example.edu.cn:8443
|
||||
# ClickHouseAnalytics__Database=jiaowu_analytics
|
||||
# ClickHouseAnalytics__UserName=jiaowu_analytics
|
||||
# ClickHouseAnalytics__Password=REPLACE_WITH_A_STRONG_PASSWORD
|
||||
ClickHouseAnalytics__CreateSchemaOnStartup=true
|
||||
ClickHouseAnalytics__SyncIntervalSeconds=60
|
||||
ClickHouseAnalytics__SourceLookbackDays=90
|
||||
ClickHouseAnalytics__BatchSize=1000
|
||||
|
||||
# 运维控制台备份目录必须位于持久化、仅服务账号可写的位置。
|
||||
Operations__BackupDirectory=/var/lib/jiaowu/backups
|
||||
Operations__BackupWarningHours=24
|
||||
|
||||
@@ -415,6 +415,22 @@ Prometheus 兼容接口,令牌应仅具有查询权限。未启用、未配置
|
||||
指标名或 `service_name` 标签做了转换,可通过 `PerformanceReporting` 下对应的
|
||||
`*MetricName` 和 `ServiceNameLabel` 配置项适配,无需改前端。
|
||||
|
||||
### ClickHouse 分析读模型
|
||||
|
||||
ClickHouse 仅用于考勤、操作审计和成绩趋势的多维聚合,MySQL 仍是所有教务业务的唯一写入源。默认关闭;启用后,后台工作器以可重试的滚动窗口投影 MySQL 当前事实到 `ReplacingMergeTree` 表,重复投递不会改变读结果。
|
||||
|
||||
生产环境请为分析库创建独立账号,并限制其只能访问 `ClickHouseAnalytics__Database`。推荐通过 HTTPS 或内网连接:
|
||||
|
||||
```ini
|
||||
ClickHouseAnalytics__Enabled=true
|
||||
ClickHouseAnalytics__Endpoint=https://clickhouse.example.edu.cn:8443
|
||||
ClickHouseAnalytics__Database=jiaowu_analytics
|
||||
ClickHouseAnalytics__UserName=jiaowu_analytics
|
||||
ClickHouseAnalytics__Password=REPLACE_WITH_A_STRONG_PASSWORD
|
||||
```
|
||||
|
||||
分析概览通过 `GET /api/clickhouse-analytics/overview` 提供;学院管理员只能读取本学院的考勤和成绩趋势,跨学院的操作审计仅对全校数据范围角色开放。ClickHouse 暂时不可用时,业务写入不会失败,工作器会在下一个周期重试。
|
||||
|
||||
### 后台任务与 RabbitMQ
|
||||
|
||||
自动排课、课表发布和补考自动生成使用数据库 Outbox 保存任务消息。创建业务任务与
|
||||
|
||||
@@ -16,6 +16,11 @@ x-jiaowu-environment: &jiaowu-environment
|
||||
ConnectionStrings__Redis: "redis:6379,abortConnect=false"
|
||||
Cache__Enabled: "true"
|
||||
Cache__KeyPrefix: "jiaowu:v1"
|
||||
ClickHouseAnalytics__Enabled: "true"
|
||||
ClickHouseAnalytics__Endpoint: "http://clickhouse:8123"
|
||||
ClickHouseAnalytics__Database: "jiaowu_analytics"
|
||||
ClickHouseAnalytics__UserName: "${CLICKHOUSE_USER:-jiaowu_analytics}"
|
||||
ClickHouseAnalytics__Password: "${CLICKHOUSE_PASSWORD:?请在 .env.docker 中设置 CLICKHOUSE_PASSWORD}"
|
||||
BackgroundJobs__Transport: RabbitMq
|
||||
BackgroundJobs__AutomaticScheduleConcurrency: "${BACKGROUND_JOB_AUTOMATIC_SCHEDULE_CONCURRENCY:-1}"
|
||||
BackgroundJobs__SchedulePublishConcurrency: "${BACKGROUND_JOB_SCHEDULE_PUBLISH_CONCURRENCY:-1}"
|
||||
@@ -47,6 +52,25 @@ x-json-logging: &json-logging
|
||||
max-file: "3"
|
||||
|
||||
services:
|
||||
clickhouse:
|
||||
image: clickhouse/clickhouse-server:25.8-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
CLICKHOUSE_DB: jiaowu_analytics
|
||||
CLICKHOUSE_USER: "${CLICKHOUSE_USER:-jiaowu_analytics}"
|
||||
CLICKHOUSE_PASSWORD: "${CLICKHOUSE_PASSWORD:?请在 .env.docker 中设置 CLICKHOUSE_PASSWORD}"
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- wget -qO- http://localhost:8123/ping | grep -q Ok
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
start_period: 20s
|
||||
volumes:
|
||||
- clickhouse-data:/var/lib/clickhouse
|
||||
logging: *json-logging
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:4.2-management-alpine
|
||||
restart: unless-stopped
|
||||
@@ -135,6 +159,8 @@ services:
|
||||
condition: service_started
|
||||
mysql:
|
||||
condition: service_healthy
|
||||
clickhouse:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
ports:
|
||||
@@ -165,5 +191,6 @@ services:
|
||||
|
||||
volumes:
|
||||
mysql-data:
|
||||
clickhouse-data:
|
||||
rabbitmq-data:
|
||||
backup-data:
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
using Jiaowu.Api.Domain.Identity;
|
||||
using Jiaowu.Api.Infrastructure.Analytics;
|
||||
using Jiaowu.Api.Infrastructure.Auth;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
namespace Jiaowu.Api.Controllers;
|
||||
|
||||
[ApiController]
|
||||
[Authorize(Roles = AnalyticsUsers)]
|
||||
[Route("api/clickhouse-analytics")]
|
||||
public sealed class ClickHouseAnalyticsController(
|
||||
ClickHouseAnalyticsClient client,
|
||||
ClickHouseAnalyticsOptions options,
|
||||
ICurrentUserDataScope currentUserDataScope) : ControllerBase
|
||||
{
|
||||
private const string AnalyticsUsers =
|
||||
SystemRoles.SuperAdmin + "," + SystemRoles.AcademicAdmin + "," +
|
||||
SystemRoles.CollegeAdmin + "," + SystemRoles.Leader;
|
||||
|
||||
[HttpGet("status")]
|
||||
public ActionResult GetStatus() => Ok(new
|
||||
{
|
||||
options.Enabled,
|
||||
options.SyncIntervalSeconds,
|
||||
options.SourceLookbackDays,
|
||||
options.BatchSize,
|
||||
options.Database
|
||||
});
|
||||
|
||||
[HttpGet("overview")]
|
||||
public async Task<ActionResult> GetOverview(
|
||||
DateOnly? from,
|
||||
DateOnly? to,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.Enabled)
|
||||
return Conflict(new ProblemDetails
|
||||
{
|
||||
Title = "ClickHouse 分析未启用",
|
||||
Detail = "请先配置 ClickHouseAnalytics 并启动分析库。",
|
||||
Status = StatusCodes.Status409Conflict
|
||||
});
|
||||
|
||||
var end = to ?? DateOnly.FromDateTime(DateTime.UtcNow);
|
||||
var start = from ?? end.AddDays(-29);
|
||||
if (start > end || end.DayNumber - start.DayNumber > 366)
|
||||
return ValidationProblem("分析时间范围应为 1 到 366 天,且开始日期不能晚于结束日期。");
|
||||
|
||||
var scope = currentUserDataScope.Current;
|
||||
var collegeFilter = scope.RestrictedCollegeId is { } collegeId
|
||||
? $" AND collegeId = toUUID('{collegeId:D}')"
|
||||
: string.Empty;
|
||||
var dateFilter = $"attendanceDate >= toDate('{start:yyyy-MM-dd}') AND attendanceDate <= toDate('{end:yyyy-MM-dd}')";
|
||||
|
||||
var attendance = await client.QueryAsync($"""
|
||||
SELECT attendanceDate, count() AS total, countIf(status = 1) AS present,
|
||||
countIf(status = 2) AS absent, countIf(status = 3) AS late
|
||||
FROM {options.Database}.attendanceRecords FINAL
|
||||
WHERE {dateFilter}{collegeFilter}
|
||||
GROUP BY attendanceDate ORDER BY attendanceDate
|
||||
""", cancellationToken);
|
||||
var grades = await client.QueryAsync($"""
|
||||
SELECT academicTermId, any(academicTermName) AS academicTermName,
|
||||
sum(studentCount) AS studentCount,
|
||||
round(sum(averageScore * studentCount) / nullIf(sum(studentCount), 0), 2) AS averageScore,
|
||||
round(sum(passedCount) / nullIf(sum(studentCount), 0), 4) AS passRate
|
||||
FROM {options.Database}.gradeStatistics FINAL
|
||||
WHERE 1 = 1{collegeFilter}
|
||||
GROUP BY academicTermId ORDER BY academicTermName
|
||||
""", cancellationToken);
|
||||
|
||||
// Audit data has no college dimension, so it is never exposed to a
|
||||
// college-scoped administrator.
|
||||
var audit = scope.RestrictedCollegeId is null
|
||||
? await client.QueryAsync($"""
|
||||
SELECT toDate(occurredAt) AS date, count() AS total,
|
||||
countIf(statusCode >= 400) AS failed
|
||||
FROM {options.Database}.auditEvents FINAL
|
||||
WHERE occurredAt >= toDateTime('{start:yyyy-MM-dd}')
|
||||
AND occurredAt < toDateTime('{end.AddDays(1):yyyy-MM-dd}')
|
||||
GROUP BY date ORDER BY date
|
||||
""", cancellationToken)
|
||||
: [];
|
||||
|
||||
return Ok(new { Start = start, End = end, Attendance = attendance, Grades = grades, Audit = audit });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace Jiaowu.Api.Infrastructure.Analytics;
|
||||
|
||||
public sealed class ClickHouseAnalyticsClient(
|
||||
HttpClient httpClient,
|
||||
ClickHouseAnalyticsOptions options)
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
public bool IsEnabled => options.Enabled;
|
||||
|
||||
public async Task EnsureSchemaAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await ExecuteAsync($"CREATE DATABASE IF NOT EXISTS {options.Database}", cancellationToken);
|
||||
|
||||
await ExecuteAsync($"""
|
||||
CREATE TABLE IF NOT EXISTS {options.Database}.auditEvents
|
||||
(
|
||||
id UUID,
|
||||
occurredAt DateTime64(3, 'UTC'),
|
||||
userId Nullable(UUID),
|
||||
method LowCardinality(String),
|
||||
path String,
|
||||
statusCode UInt16,
|
||||
ipAddress Nullable(String),
|
||||
projectedAt DateTime64(3, 'UTC')
|
||||
) ENGINE = ReplacingMergeTree(projectedAt)
|
||||
PARTITION BY toYYYYMM(occurredAt)
|
||||
ORDER BY (id)
|
||||
""", cancellationToken);
|
||||
|
||||
await ExecuteAsync($"""
|
||||
CREATE TABLE IF NOT EXISTS {options.Database}.attendanceRecords
|
||||
(
|
||||
attendanceSheetId UUID,
|
||||
studentId UUID,
|
||||
attendanceDate Date,
|
||||
teachingTaskId UUID,
|
||||
academicTermId UUID,
|
||||
collegeId UUID,
|
||||
status UInt8,
|
||||
checkInAt Nullable(DateTime64(3, 'UTC')),
|
||||
checkedInMethod Nullable(UInt8),
|
||||
appealStatus UInt8,
|
||||
projectedAt DateTime64(3, 'UTC')
|
||||
) ENGINE = ReplacingMergeTree(projectedAt)
|
||||
PARTITION BY toYYYYMM(attendanceDate)
|
||||
ORDER BY (attendanceSheetId, studentId)
|
||||
""", cancellationToken);
|
||||
|
||||
await ExecuteAsync($"""
|
||||
CREATE TABLE IF NOT EXISTS {options.Database}.gradeStatistics
|
||||
(
|
||||
gradeSheetId UUID,
|
||||
teachingTaskId UUID,
|
||||
courseId UUID,
|
||||
academicTermId UUID,
|
||||
collegeId UUID,
|
||||
academicTermName LowCardinality(String),
|
||||
studentCount UInt32,
|
||||
passedCount UInt32,
|
||||
excellentCount UInt32,
|
||||
averageScore Decimal(8, 2),
|
||||
passRate Decimal(8, 4),
|
||||
excellentRate Decimal(8, 4),
|
||||
calculatedAt DateTime64(3, 'UTC'),
|
||||
projectedAt DateTime64(3, 'UTC')
|
||||
) ENGINE = ReplacingMergeTree(projectedAt)
|
||||
PARTITION BY toYYYYMM(calculatedAt)
|
||||
ORDER BY (gradeSheetId)
|
||||
""", cancellationToken);
|
||||
}
|
||||
|
||||
public async Task InsertAsync<T>(string table, IReadOnlyCollection<T> rows, CancellationToken cancellationToken)
|
||||
{
|
||||
if (rows.Count == 0) return;
|
||||
var payload = JsonSerializer.Serialize(rows, JsonOptions);
|
||||
await ExecuteAsync(
|
||||
$"INSERT INTO {options.Database}.{table} FORMAT JSONEachRow\n{ToJsonLines(payload)}",
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<JsonElement[]> QueryAsync(string sql, CancellationToken cancellationToken)
|
||||
{
|
||||
using var response = await SendAsync(sql + " FORMAT JSON", cancellationToken);
|
||||
var content = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
if (!response.IsSuccessStatusCode)
|
||||
throw new HttpRequestException($"ClickHouse 查询失败 ({(int)response.StatusCode}):{content}");
|
||||
using var document = JsonDocument.Parse(content);
|
||||
return document.RootElement.GetProperty("data")
|
||||
.EnumerateArray().Select(x => x.Clone()).ToArray();
|
||||
}
|
||||
|
||||
private async Task ExecuteAsync(string sql, CancellationToken cancellationToken)
|
||||
{
|
||||
using var response = await SendAsync(sql, cancellationToken);
|
||||
if (response.IsSuccessStatusCode) return;
|
||||
var content = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
throw new HttpRequestException($"ClickHouse 写入失败 ({(int)response.StatusCode}):{content}");
|
||||
}
|
||||
|
||||
private Task<HttpResponseMessage> SendAsync(string sql, CancellationToken cancellationToken)
|
||||
{
|
||||
var request = new HttpRequestMessage(HttpMethod.Post, "")
|
||||
{
|
||||
Content = new StringContent(sql, Encoding.UTF8, "text/plain")
|
||||
};
|
||||
var credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes(
|
||||
$"{options.UserName}:{options.Password}"));
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Basic", credentials);
|
||||
return httpClient.SendAsync(request, cancellationToken);
|
||||
}
|
||||
|
||||
private static string ToJsonLines(string json)
|
||||
{
|
||||
using var document = JsonDocument.Parse(json);
|
||||
return string.Join('\n', document.RootElement.EnumerateArray().Select(x => x.GetRawText()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace Jiaowu.Api.Infrastructure.Analytics;
|
||||
|
||||
public sealed partial class ClickHouseAnalyticsOptions
|
||||
{
|
||||
public const string SectionName = "ClickHouseAnalytics";
|
||||
|
||||
public bool Enabled { get; set; }
|
||||
public string Endpoint { get; set; } = "http://localhost:8123";
|
||||
public string Database { get; set; } = "jiaowu_analytics";
|
||||
public string UserName { get; set; } = "jiaowu_analytics";
|
||||
public string Password { get; set; } = "";
|
||||
public bool CreateSchemaOnStartup { get; set; } = true;
|
||||
public int SyncIntervalSeconds { get; set; } = 60;
|
||||
public int SourceLookbackDays { get; set; } = 90;
|
||||
public int BatchSize { get; set; } = 1000;
|
||||
|
||||
[GeneratedRegex("^[a-zA-Z_][a-zA-Z0-9_]{0,62}$")]
|
||||
private static partial Regex IdentifierPattern();
|
||||
|
||||
public bool HasValidIdentifiers() =>
|
||||
IdentifierPattern().IsMatch(Database);
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
using Jiaowu.Api.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace Jiaowu.Api.Infrastructure.Analytics;
|
||||
|
||||
/// <summary>
|
||||
/// Projects MySQL facts to ClickHouse. The projection is deliberately
|
||||
/// best-effort: business writes never depend on an analytics database.
|
||||
/// ReplacingMergeTree plus FINAL reads make repeated lookback batches safe.
|
||||
/// </summary>
|
||||
public sealed class ClickHouseAnalyticsProjectionWorker(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
ClickHouseAnalyticsClient client,
|
||||
ClickHouseAnalyticsOptions options,
|
||||
ILogger<ClickHouseAnalyticsProjectionWorker> logger) : BackgroundService
|
||||
{
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
if (!options.Enabled) return;
|
||||
try
|
||||
{
|
||||
if (options.CreateSchemaOnStartup)
|
||||
await client.EnsureSchemaAsync(stoppingToken);
|
||||
}
|
||||
catch (Exception exception) when (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
logger.LogError(exception, "ClickHouse 分析表初始化失败,将在下一轮重试。");
|
||||
}
|
||||
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(options.SyncIntervalSeconds));
|
||||
do
|
||||
{
|
||||
try
|
||||
{
|
||||
await ProjectAsync(stoppingToken);
|
||||
}
|
||||
catch (Exception exception) when (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
logger.LogError(exception, "ClickHouse 分析投影失败,将在下一轮重试。");
|
||||
}
|
||||
} while (await timer.WaitForNextTickAsync(stoppingToken));
|
||||
}
|
||||
|
||||
private async Task ProjectAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var projectedAt = DateTime.UtcNow;
|
||||
var from = projectedAt.AddDays(-options.SourceLookbackDays);
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
|
||||
var auditCount = await ProjectAuditsAsync(db, from, projectedAt, cancellationToken);
|
||||
var attendanceCount = await ProjectAttendanceAsync(db, from.Date, projectedAt, cancellationToken);
|
||||
var gradeCount = await ProjectGradesAsync(db, projectedAt, cancellationToken);
|
||||
logger.LogInformation(
|
||||
"ClickHouse 分析投影完成:审计 {AuditCount},考勤 {AttendanceCount},成绩 {GradeCount}。",
|
||||
auditCount, attendanceCount, gradeCount);
|
||||
}
|
||||
|
||||
private async Task<int> ProjectAuditsAsync(AppDbContext db, DateTime from, DateTime projectedAt, CancellationToken cancellationToken)
|
||||
{
|
||||
var cursorAt = from;
|
||||
var cursorId = Guid.Empty;
|
||||
var count = 0;
|
||||
while (true)
|
||||
{
|
||||
var rows = await db.AuditLogs.AsNoTracking()
|
||||
.Where(x => x.CreatedAt > cursorAt || x.CreatedAt == cursorAt && x.Id.CompareTo(cursorId) > 0)
|
||||
.OrderBy(x => x.CreatedAt).ThenBy(x => x.Id).Take(options.BatchSize)
|
||||
.Select(x => new { x.Id, OccurredAt = x.CreatedAt, x.UserId, x.Method, x.Path, x.StatusCode, x.IpAddress, ProjectedAt = projectedAt })
|
||||
.ToListAsync(cancellationToken);
|
||||
if (rows.Count == 0) return count;
|
||||
await client.InsertAsync("auditEvents", rows, cancellationToken);
|
||||
count += rows.Count;
|
||||
cursorAt = rows[^1].OccurredAt;
|
||||
cursorId = rows[^1].Id;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<int> ProjectAttendanceAsync(AppDbContext db, DateTime from, DateTime projectedAt, CancellationToken cancellationToken)
|
||||
{
|
||||
var cursorDate = from;
|
||||
var cursorSheetId = Guid.Empty;
|
||||
var cursorStudentId = Guid.Empty;
|
||||
var count = 0;
|
||||
while (true)
|
||||
{
|
||||
var rows = await db.AttendanceRecords.AsNoTracking()
|
||||
.Where(x => x.AttendanceSheet!.AttendanceDate > cursorDate ||
|
||||
x.AttendanceSheet.AttendanceDate == cursorDate &&
|
||||
(x.AttendanceSheetId.CompareTo(cursorSheetId) > 0 ||
|
||||
x.AttendanceSheetId == cursorSheetId && x.StudentId.CompareTo(cursorStudentId) > 0))
|
||||
.OrderBy(x => x.AttendanceSheet!.AttendanceDate).ThenBy(x => x.AttendanceSheetId).ThenBy(x => x.StudentId).Take(options.BatchSize)
|
||||
.Select(x => new { x.AttendanceSheetId, x.StudentId, AttendanceDate = x.AttendanceSheet!.AttendanceDate, TeachingTaskId = x.AttendanceSheet.TeachingTaskId, AcademicTermId = x.AttendanceSheet.TeachingTask!.AcademicTermId, CollegeId = x.AttendanceSheet.TeachingTask.Course!.CollegeId, Status = (byte)x.Status, x.CheckInAt, CheckedInMethod = x.CheckedInMethod == null ? null : (byte?)x.CheckedInMethod, AppealStatus = (byte)x.AppealStatus, ProjectedAt = projectedAt })
|
||||
.ToListAsync(cancellationToken);
|
||||
if (rows.Count == 0) return count;
|
||||
await client.InsertAsync("attendanceRecords", rows, cancellationToken);
|
||||
count += rows.Count;
|
||||
cursorDate = rows[^1].AttendanceDate;
|
||||
cursorSheetId = rows[^1].AttendanceSheetId;
|
||||
cursorStudentId = rows[^1].StudentId;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<int> ProjectGradesAsync(AppDbContext db, DateTime projectedAt, CancellationToken cancellationToken)
|
||||
{
|
||||
var cursorAt = DateTime.MinValue;
|
||||
var cursorSheetId = Guid.Empty;
|
||||
var count = 0;
|
||||
while (true)
|
||||
{
|
||||
var rows = await db.TeachingTaskGradeStatistics.AsNoTracking()
|
||||
.Where(x => x.CalculatedAt > cursorAt || x.CalculatedAt == cursorAt && x.GradeSheetId.CompareTo(cursorSheetId) > 0)
|
||||
.OrderBy(x => x.CalculatedAt).ThenBy(x => x.GradeSheetId).Take(options.BatchSize)
|
||||
.Select(x => new { x.GradeSheetId, x.TeachingTaskId, x.CourseId, x.AcademicTermId, CollegeId = x.TeachingTask!.Course!.CollegeId, AcademicTermName = x.TeachingTask.AcademicTerm!.Name, x.StudentCount, x.PassedCount, x.ExcellentCount, x.AverageScore, x.PassRate, x.ExcellentRate, x.CalculatedAt, ProjectedAt = projectedAt })
|
||||
.ToListAsync(cancellationToken);
|
||||
if (rows.Count == 0) return count;
|
||||
await client.InsertAsync("gradeStatistics", rows, cancellationToken);
|
||||
count += rows.Count;
|
||||
cursorAt = rows[^1].CalculatedAt;
|
||||
cursorSheetId = rows[^1].GradeSheetId;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ using Jiaowu.Api.Infrastructure.BackgroundJobs;
|
||||
using Jiaowu.Api.Infrastructure.Grades;
|
||||
using Jiaowu.Api.Infrastructure.Configuration;
|
||||
using Jiaowu.Api.Infrastructure.Auth;
|
||||
using Jiaowu.Api.Infrastructure.Analytics;
|
||||
using Jiaowu.Api.Infrastructure.Caching;
|
||||
using Jiaowu.Api.Infrastructure.Exams;
|
||||
using Jiaowu.Api.Infrastructure.Middleware;
|
||||
@@ -101,6 +102,9 @@ var observabilityOptions = builder.Configuration
|
||||
var performanceReportingOptions = builder.Configuration
|
||||
.GetSection(PerformanceReportingOptions.SectionName)
|
||||
.Get<PerformanceReportingOptions>() ?? new PerformanceReportingOptions();
|
||||
var clickHouseAnalyticsOptions = builder.Configuration
|
||||
.GetSection(ClickHouseAnalyticsOptions.SectionName)
|
||||
.Get<ClickHouseAnalyticsOptions>() ?? new ClickHouseAnalyticsOptions();
|
||||
var rabbitMqOptions = builder.Configuration
|
||||
.GetSection(RabbitMqOptions.SectionName)
|
||||
.Get<RabbitMqOptions>() ?? new RabbitMqOptions();
|
||||
@@ -252,6 +256,19 @@ if (backgroundJobOptions.UsesRabbitMq &&
|
||||
{
|
||||
throw new InvalidOperationException("RabbitMq 连接配置不完整。");
|
||||
}
|
||||
|
||||
if (clickHouseAnalyticsOptions.Enabled &&
|
||||
(!Uri.TryCreate(clickHouseAnalyticsOptions.Endpoint, UriKind.Absolute, out var clickHouseEndpoint) ||
|
||||
clickHouseEndpoint.Scheme is not ("http" or "https") ||
|
||||
!clickHouseAnalyticsOptions.HasValidIdentifiers() ||
|
||||
string.IsNullOrWhiteSpace(clickHouseAnalyticsOptions.UserName) ||
|
||||
string.IsNullOrWhiteSpace(clickHouseAnalyticsOptions.Password) ||
|
||||
clickHouseAnalyticsOptions.SyncIntervalSeconds is < 10 or > 86400 ||
|
||||
clickHouseAnalyticsOptions.SourceLookbackDays is < 1 or > 3650 ||
|
||||
clickHouseAnalyticsOptions.BatchSize is < 1 or > 10000))
|
||||
{
|
||||
throw new InvalidOperationException("ClickHouseAnalytics 配置无效。");
|
||||
}
|
||||
if (backgroundJobOptions.UsesRabbitMq &&
|
||||
!builder.Environment.IsDevelopment() &&
|
||||
(rabbitMqOptions.UserName.Equals("guest", StringComparison.OrdinalIgnoreCase) ||
|
||||
@@ -282,6 +299,7 @@ builder.Services.AddSingleton(backgroundJobOptions);
|
||||
builder.Services.AddSingleton(operationsOptions);
|
||||
builder.Services.AddSingleton(observabilityOptions);
|
||||
builder.Services.AddSingleton(performanceReportingOptions);
|
||||
builder.Services.AddSingleton(clickHouseAnalyticsOptions);
|
||||
builder.Services.AddSingleton(rabbitMqOptions);
|
||||
builder.Services.AddSingleton<DatabaseCommandTelemetryInterceptor>();
|
||||
builder.Services.AddMemoryCache();
|
||||
@@ -290,6 +308,11 @@ builder.Services.AddHttpClient<PerformanceReportService>((services, client) =>
|
||||
var reporting = services.GetRequiredService<PerformanceReportingOptions>();
|
||||
client.Timeout = TimeSpan.FromSeconds(reporting.TimeoutSeconds);
|
||||
});
|
||||
builder.Services.AddHttpClient<ClickHouseAnalyticsClient>((_, client) =>
|
||||
{
|
||||
client.BaseAddress = new Uri(clickHouseAnalyticsOptions.Endpoint.TrimEnd('/') + "/");
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
builder.Services.Configure<OfficialDocumentOptions>(
|
||||
builder.Configuration.GetSection(OfficialDocumentOptions.SectionName));
|
||||
builder.Services.AddDbContextPool<AppDbContext>((services, options) =>
|
||||
@@ -441,6 +464,7 @@ builder.Services.AddScoped<CourseGradeStatisticsRefreshJobProcessor>();
|
||||
builder.Services.AddScoped<CourseGradeStatisticsRefreshScheduler>();
|
||||
builder.Services.AddSingleton(TimeProvider.System);
|
||||
builder.Services.AddHostedService<CourseGradeStatisticsRefreshWorker>();
|
||||
builder.Services.AddHostedService<ClickHouseAnalyticsProjectionWorker>();
|
||||
builder.Services.AddSingleton<BackgroundJobTelemetry>();
|
||||
builder.Services.AddScoped<BackgroundJobMonitoringService>();
|
||||
builder.Services.AddScoped<OperationalHealthService>();
|
||||
|
||||
@@ -39,6 +39,17 @@
|
||||
"SlowDatabaseMetric": "jiaowu_db_command_slow_total",
|
||||
"FailedDatabaseMetric": "jiaowu_db_command_failed_total"
|
||||
},
|
||||
"ClickHouseAnalytics": {
|
||||
"Enabled": false,
|
||||
"Endpoint": "http://localhost:8123",
|
||||
"Database": "jiaowu_analytics",
|
||||
"UserName": "jiaowu_analytics",
|
||||
"Password": "",
|
||||
"CreateSchemaOnStartup": true,
|
||||
"SyncIntervalSeconds": 60,
|
||||
"SourceLookbackDays": 90,
|
||||
"BatchSize": 1000
|
||||
},
|
||||
"Operations": {
|
||||
"BackupDirectory": "data/backups",
|
||||
"BackupWarningHours": 24,
|
||||
|
||||
@@ -194,6 +194,10 @@ const navigationGroups = computed<NavigationGroup[]>(() => [
|
||||
hasAnyRole(['SuperAdmin', 'AcademicAdmin', 'CollegeAdmin', 'Leader', 'Teacher']),
|
||||
{ path: '/grade-analytics', label: isTeacher.value ? '教学班成绩分析' : '成绩分析中心' },
|
||||
),
|
||||
...whenVisible(
|
||||
hasAnyRole(['SuperAdmin', 'AcademicAdmin', 'CollegeAdmin', 'Leader']),
|
||||
{ path: '/event-analytics', label: '运行数据分析' },
|
||||
),
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
@@ -263,6 +263,14 @@ const router = createRouter({
|
||||
roles: ['SuperAdmin', 'AcademicAdmin', 'CollegeAdmin', 'Leader', 'Teacher'],
|
||||
},
|
||||
},
|
||||
{
|
||||
path: 'event-analytics',
|
||||
name: 'event-analytics',
|
||||
component: () => import('../views/EventAnalyticsView.vue'),
|
||||
meta: {
|
||||
roles: ['SuperAdmin', 'AcademicAdmin', 'CollegeAdmin', 'Leader'],
|
||||
},
|
||||
},
|
||||
{
|
||||
path: 'other-exams',
|
||||
name: 'other-exams',
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, onMounted, ref } from 'vue'
|
||||
import { DataAnalysis, Refresh } from '@element-plus/icons-vue'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import http, { apiErrorMessage } from '../api/http'
|
||||
|
||||
interface Status {
|
||||
enabled: boolean
|
||||
syncIntervalSeconds: number
|
||||
sourceLookbackDays: number
|
||||
batchSize: number
|
||||
database: string
|
||||
}
|
||||
|
||||
const status = ref<Status>()
|
||||
const attendance = ref<any[]>([])
|
||||
const grades = ref<any[]>([])
|
||||
const audit = ref<any[]>([])
|
||||
const loading = ref(false)
|
||||
const range = ref<[Date, Date]>([
|
||||
new Date(Date.now() - 29 * 24 * 60 * 60 * 1000),
|
||||
new Date(),
|
||||
])
|
||||
|
||||
const attendanceTotals = computed(() => attendance.value.reduce((total, row) => total + Number(row.total ?? 0), 0))
|
||||
const absentTotals = computed(() => attendance.value.reduce((total, row) => total + Number(row.absent ?? 0), 0))
|
||||
const auditTotals = computed(() => audit.value.reduce((total, row) => total + Number(row.total ?? 0), 0))
|
||||
|
||||
function dateOnly(value: Date) {
|
||||
const offset = value.getTimezoneOffset() * 60_000
|
||||
return new Date(value.getTime() - offset).toISOString().slice(0, 10)
|
||||
}
|
||||
|
||||
async function load() {
|
||||
loading.value = true
|
||||
try {
|
||||
const [statusResponse, overviewResponse] = await Promise.all([
|
||||
http.get<Status>('/clickhouse-analytics/status'),
|
||||
http.get('/clickhouse-analytics/overview', {
|
||||
params: { from: dateOnly(range.value[0]), to: dateOnly(range.value[1]) },
|
||||
}),
|
||||
])
|
||||
status.value = statusResponse.data
|
||||
attendance.value = overviewResponse.data.attendance ?? []
|
||||
grades.value = overviewResponse.data.grades ?? []
|
||||
audit.value = overviewResponse.data.audit ?? []
|
||||
} catch (error) {
|
||||
ElMessage.error(apiErrorMessage(error) || '加载运行数据分析失败。')
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(load)
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<main v-loading="loading" class="event-analytics page-stack">
|
||||
<section class="page-intro">
|
||||
<div>
|
||||
<span class="eyebrow">CLICKHOUSE READ MODEL</span>
|
||||
<h2>运行数据分析</h2>
|
||||
<p>考勤、教学班成绩与访问审计的只读聚合;不会影响教务业务写入。</p>
|
||||
</div>
|
||||
<div class="actions">
|
||||
<el-date-picker v-model="range" type="daterange" range-separator="至" start-placeholder="开始日期" end-placeholder="结束日期" :clearable="false" />
|
||||
<el-button type="primary" :icon="Refresh" @click="load">刷新</el-button>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<el-alert v-if="status && !status.enabled" type="warning" :closable="false" show-icon title="ClickHouse 分析未启用">
|
||||
请在服务配置中启用 ClickHouseAnalytics 后再查看分析数据。
|
||||
</el-alert>
|
||||
|
||||
<template v-else>
|
||||
<section class="metrics">
|
||||
<article><el-icon><DataAnalysis /></el-icon><span>考勤记录</span><b>{{ attendanceTotals.toLocaleString() }}</b></article>
|
||||
<article><el-icon><DataAnalysis /></el-icon><span>缺勤记录</span><b>{{ absentTotals.toLocaleString() }}</b></article>
|
||||
<article v-if="audit.length"><el-icon><DataAnalysis /></el-icon><span>访问审计</span><b>{{ auditTotals.toLocaleString() }}</b></article>
|
||||
</section>
|
||||
|
||||
<section class="analysis-grid">
|
||||
<el-card shadow="never">
|
||||
<template #header>每日考勤</template>
|
||||
<el-table :data="attendance" size="small" empty-text="所选范围暂无考勤投影数据">
|
||||
<el-table-column prop="attendanceDate" label="日期" min-width="110" />
|
||||
<el-table-column prop="total" label="总人次" align="right" />
|
||||
<el-table-column prop="present" label="到课" align="right" />
|
||||
<el-table-column prop="absent" label="缺勤" align="right" />
|
||||
<el-table-column prop="late" label="迟到" align="right" />
|
||||
</el-table>
|
||||
</el-card>
|
||||
<el-card shadow="never">
|
||||
<template #header>学期成绩趋势</template>
|
||||
<el-table :data="grades" size="small" empty-text="暂无已计算的教学班成绩统计">
|
||||
<el-table-column prop="academicTermName" label="学期" min-width="130" />
|
||||
<el-table-column prop="studentCount" label="学生数" align="right" />
|
||||
<el-table-column prop="averageScore" label="加权平均分" align="right" />
|
||||
<el-table-column prop="passRate" label="通过率" align="right">
|
||||
<template #default="{ row }">{{ (Number(row.passRate) * 100).toFixed(1) }}%</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
</el-card>
|
||||
</section>
|
||||
|
||||
<el-card v-if="audit.length" shadow="never">
|
||||
<template #header>访问审计(仅全校数据范围)</template>
|
||||
<el-table :data="audit" size="small">
|
||||
<el-table-column prop="date" label="日期" min-width="110" />
|
||||
<el-table-column prop="total" label="操作量" align="right" />
|
||||
<el-table-column prop="failed" label="异常响应" align="right" />
|
||||
</el-table>
|
||||
</el-card>
|
||||
</template>
|
||||
</main>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.event-analytics { --ink: #18324f; --line: #dce3ec; }
|
||||
.page-intro { display: flex; justify-content: space-between; align-items: end; gap: 18px; }
|
||||
.eyebrow { color: #3d75aa; font-size: 12px; letter-spacing: .12em; font-weight: 700; }
|
||||
h2 { margin: 5px 0; color: var(--ink); } p { margin: 0; color: #6b7b8d; }
|
||||
.actions { display: flex; gap: 10px; flex-wrap: wrap; }
|
||||
.metrics, .analysis-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 16px; }
|
||||
.analysis-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||
.metrics article { padding: 20px; border: 1px solid var(--line); background: #fff; border-radius: 8px; display: grid; grid-template-columns: 26px 1fr; gap: 4px 9px; }
|
||||
.metrics .el-icon { color: #3574a8; grid-row: span 2; font-size: 21px; } .metrics span { color: #657689; font-size: 13px; } .metrics b { color: var(--ink); font-size: 24px; }
|
||||
@media (max-width: 760px) { .page-intro { align-items: start; flex-direction: column; } .metrics, .analysis-grid { grid-template-columns: 1fr; } }
|
||||
</style>
|
||||
Reference in New Issue
Block a user