clickhouse

src/Jiaowu.Api/Controllers/ClickHouseAnalyticsController.cs

@@ -0,0 +1,88 @@
+using Jiaowu.Api.Domain.Identity;
+using Jiaowu.Api.Infrastructure.Analytics;
+using Jiaowu.Api.Infrastructure.Auth;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+
+namespace Jiaowu.Api.Controllers;
+
+[ApiController]
+[Authorize(Roles = AnalyticsUsers)]
+[Route("api/clickhouse-analytics")]
+public sealed class ClickHouseAnalyticsController(
+    ClickHouseAnalyticsClient client,
+    ClickHouseAnalyticsOptions options,
+    ICurrentUserDataScope currentUserDataScope) : ControllerBase
+{
+    private const string AnalyticsUsers =
+        SystemRoles.SuperAdmin + "," + SystemRoles.AcademicAdmin + "," +
+        SystemRoles.CollegeAdmin + "," + SystemRoles.Leader;
+
+    [HttpGet("status")]
+    public ActionResult GetStatus() => Ok(new
+    {
+        options.Enabled,
+        options.SyncIntervalSeconds,
+        options.SourceLookbackDays,
+        options.BatchSize,
+        options.Database
+    });
+
+    [HttpGet("overview")]
+    public async Task<ActionResult> GetOverview(
+        DateOnly? from,
+        DateOnly? to,
+        CancellationToken cancellationToken)
+    {
+        if (!options.Enabled)
+            return Conflict(new ProblemDetails
+            {
+                Title = "ClickHouse 分析未启用",
+                Detail = "请先配置 ClickHouseAnalytics 并启动分析库。",
+                Status = StatusCodes.Status409Conflict
+            });
+
+        var end = to ?? DateOnly.FromDateTime(DateTime.UtcNow);
+        var start = from ?? end.AddDays(-29);
+        if (start > end || end.DayNumber - start.DayNumber > 366)
+            return ValidationProblem("分析时间范围应为 1 到 366 天，且开始日期不能晚于结束日期。");
+
+        var scope = currentUserDataScope.Current;
+        var collegeFilter = scope.RestrictedCollegeId is { } collegeId
+            ? $" AND collegeId = toUUID('{collegeId:D}')"
+            : string.Empty;
+        var dateFilter = $"attendanceDate >= toDate('{start:yyyy-MM-dd}') AND attendanceDate <= toDate('{end:yyyy-MM-dd}')";
+
+        var attendance = await client.QueryAsync($"""
+            SELECT attendanceDate, count() AS total, countIf(status = 1) AS present,
+                   countIf(status = 2) AS absent, countIf(status = 3) AS late
+            FROM {options.Database}.attendanceRecords FINAL
+            WHERE {dateFilter}{collegeFilter}
+            GROUP BY attendanceDate ORDER BY attendanceDate
+            """, cancellationToken);
+        var grades = await client.QueryAsync($"""
+            SELECT academicTermId, any(academicTermName) AS academicTermName,
+                   sum(studentCount) AS studentCount,
+                   round(sum(averageScore * studentCount) / nullIf(sum(studentCount), 0), 2) AS averageScore,
+                   round(sum(passedCount) / nullIf(sum(studentCount), 0), 4) AS passRate
+            FROM {options.Database}.gradeStatistics FINAL
+            WHERE 1 = 1{collegeFilter}
+            GROUP BY academicTermId ORDER BY academicTermName
+            """, cancellationToken);
+
+        // Audit data has no college dimension, so it is never exposed to a
+        // college-scoped administrator.
+        var audit = scope.RestrictedCollegeId is null
+            ? await client.QueryAsync($"""
+                SELECT toDate(occurredAt) AS date, count() AS total,
+                       countIf(statusCode >= 400) AS failed
+                FROM {options.Database}.auditEvents FINAL
+                WHERE occurredAt >= toDateTime('{start:yyyy-MM-dd}')
+                  AND occurredAt < toDateTime('{end.AddDays(1):yyyy-MM-dd}')
+                GROUP BY date ORDER BY date
+                """, cancellationToken)
+            : [];
+
+        return Ok(new { Start = start, End = end, Attendance = attendance, Grades = grades, Audit = audit });
+    }
+}

src/Jiaowu.Api/Infrastructure/Analytics/ClickHouseAnalyticsClient.cs

@@ -0,0 +1,122 @@
+using System.Net.Http.Headers;
+using System.Text;
+using System.Text.Json;
+
+namespace Jiaowu.Api.Infrastructure.Analytics;
+
+public sealed class ClickHouseAnalyticsClient(
+    HttpClient httpClient,
+    ClickHouseAnalyticsOptions options)
+{
+    private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
+
+    public bool IsEnabled => options.Enabled;
+
+    public async Task EnsureSchemaAsync(CancellationToken cancellationToken)
+    {
+        await ExecuteAsync($"CREATE DATABASE IF NOT EXISTS {options.Database}", cancellationToken);
+
+        await ExecuteAsync($"""
+            CREATE TABLE IF NOT EXISTS {options.Database}.auditEvents
+            (
+                id UUID,
+                occurredAt DateTime64(3, 'UTC'),
+                userId Nullable(UUID),
+                method LowCardinality(String),
+                path String,
+                statusCode UInt16,
+                ipAddress Nullable(String),
+                projectedAt DateTime64(3, 'UTC')
+            ) ENGINE = ReplacingMergeTree(projectedAt)
+            PARTITION BY toYYYYMM(occurredAt)
+            ORDER BY (id)
+            """, cancellationToken);
+
+        await ExecuteAsync($"""
+            CREATE TABLE IF NOT EXISTS {options.Database}.attendanceRecords
+            (
+                attendanceSheetId UUID,
+                studentId UUID,
+                attendanceDate Date,
+                teachingTaskId UUID,
+                academicTermId UUID,
+                collegeId UUID,
+                status UInt8,
+                checkInAt Nullable(DateTime64(3, 'UTC')),
+                checkedInMethod Nullable(UInt8),
+                appealStatus UInt8,
+                projectedAt DateTime64(3, 'UTC')
+            ) ENGINE = ReplacingMergeTree(projectedAt)
+            PARTITION BY toYYYYMM(attendanceDate)
+            ORDER BY (attendanceSheetId, studentId)
+            """, cancellationToken);
+
+        await ExecuteAsync($"""
+            CREATE TABLE IF NOT EXISTS {options.Database}.gradeStatistics
+            (
+                gradeSheetId UUID,
+                teachingTaskId UUID,
+                courseId UUID,
+                academicTermId UUID,
+                collegeId UUID,
+                academicTermName LowCardinality(String),
+                studentCount UInt32,
+                passedCount UInt32,
+                excellentCount UInt32,
+                averageScore Decimal(8, 2),
+                passRate Decimal(8, 4),
+                excellentRate Decimal(8, 4),
+                calculatedAt DateTime64(3, 'UTC'),
+                projectedAt DateTime64(3, 'UTC')
+            ) ENGINE = ReplacingMergeTree(projectedAt)
+            PARTITION BY toYYYYMM(calculatedAt)
+            ORDER BY (gradeSheetId)
+            """, cancellationToken);
+    }
+
+    public async Task InsertAsync<T>(string table, IReadOnlyCollection<T> rows, CancellationToken cancellationToken)
+    {
+        if (rows.Count == 0) return;
+        var payload = JsonSerializer.Serialize(rows, JsonOptions);
+        await ExecuteAsync(
+            $"INSERT INTO {options.Database}.{table} FORMAT JSONEachRow\n{ToJsonLines(payload)}",
+            cancellationToken);
+    }
+
+    public async Task<JsonElement[]> QueryAsync(string sql, CancellationToken cancellationToken)
+    {
+        using var response = await SendAsync(sql + " FORMAT JSON", cancellationToken);
+        var content = await response.Content.ReadAsStringAsync(cancellationToken);
+        if (!response.IsSuccessStatusCode)
+            throw new HttpRequestException($"ClickHouse 查询失败 ({(int)response.StatusCode})：{content}");
+        using var document = JsonDocument.Parse(content);
+        return document.RootElement.GetProperty("data")
+            .EnumerateArray().Select(x => x.Clone()).ToArray();
+    }
+
+    private async Task ExecuteAsync(string sql, CancellationToken cancellationToken)
+    {
+        using var response = await SendAsync(sql, cancellationToken);
+        if (response.IsSuccessStatusCode) return;
+        var content = await response.Content.ReadAsStringAsync(cancellationToken);
+        throw new HttpRequestException($"ClickHouse 写入失败 ({(int)response.StatusCode})：{content}");
+    }
+
+    private Task<HttpResponseMessage> SendAsync(string sql, CancellationToken cancellationToken)
+    {
+        var request = new HttpRequestMessage(HttpMethod.Post, "")
+        {
+            Content = new StringContent(sql, Encoding.UTF8, "text/plain")
+        };
+        var credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes(
+            $"{options.UserName}:{options.Password}"));
+        request.Headers.Authorization = new AuthenticationHeaderValue("Basic", credentials);
+        return httpClient.SendAsync(request, cancellationToken);
+    }
+
+    private static string ToJsonLines(string json)
+    {
+        using var document = JsonDocument.Parse(json);
+        return string.Join('\n', document.RootElement.EnumerateArray().Select(x => x.GetRawText()));
+    }
+}

src/Jiaowu.Api/Infrastructure/Analytics/ClickHouseAnalyticsOptions.cs

@@ -0,0 +1,24 @@
+using System.Text.RegularExpressions;
+
+namespace Jiaowu.Api.Infrastructure.Analytics;
+
+public sealed partial class ClickHouseAnalyticsOptions
+{
+    public const string SectionName = "ClickHouseAnalytics";
+
+    public bool Enabled { get; set; }
+    public string Endpoint { get; set; } = "http://localhost:8123";
+    public string Database { get; set; } = "jiaowu_analytics";
+    public string UserName { get; set; } = "jiaowu_analytics";
+    public string Password { get; set; } = "";
+    public bool CreateSchemaOnStartup { get; set; } = true;
+    public int SyncIntervalSeconds { get; set; } = 60;
+    public int SourceLookbackDays { get; set; } = 90;
+    public int BatchSize { get; set; } = 1000;
+
+    [GeneratedRegex("^[a-zA-Z_][a-zA-Z0-9_]{0,62}$")]
+    private static partial Regex IdentifierPattern();
+
+    public bool HasValidIdentifiers() =>
+        IdentifierPattern().IsMatch(Database);
+}