接入 OpenTelemetry 1.17.0，覆盖 HTTP、HttpClient、.NET Runtime 和数据库链路。
新增 EF Core 数据库拦截器，记录耗时、失败数、慢查询数、TraceId、查询标签和 SQL 哈希；默认不记录完整 SQL及参数。 未配置 OTLP Collector 时不启动 SDK，避免无收益的性能开销。 为课表的作息、课程、灵活课程、考试、实验等六类查询增加稳定标签。 增加可配置的 500ms 慢查询阈值，以及生产环境变量示例。 README 补充 MySQL 慢查询与 EXPLAIN ANALYZE 操作规范。
This commit is contained in:
biss committed
1 parent
a369b4fe02
commit
ca7148ab61
9 files changed
+489
-1
No files matched your search
@@ -36,6 +36,16 @@ Cache__AnalyticsExpirationMinutes=3
|
||||
Cache__AnalyticsLocalExpirationSeconds=30
|
||||
Cache__MaximumPayloadKilobytes=2048
|
||||
|
||||
# OpenTelemetry 默认收集 HTTP、运行时和数据库指标；配置 OTLP 地址后才会外发。
|
||||
Observability__Enabled=true
|
||||
Observability__ServiceName=jiaowu-api
|
||||
Observability__SlowQueryThresholdMilliseconds=500
|
||||
# 默认不记录完整 SQL，避免日志或追踪系统接触业务数据。
|
||||
Observability__IncludeSqlText=false
|
||||
Observability__MaximumSqlTextLength=2000
|
||||
# OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector.example.edu.cn:4317
|
||||
# OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer%20REPLACE_WITH_TOKEN
|
||||
|
||||
# 运维控制台备份目录必须位于持久化、仅服务账号可写的位置。
|
||||
Operations__BackupDirectory=/var/lib/jiaowu/backups
|
||||
Operations__BackupWarningHours=24
|
||||
|
||||
@@ -322,6 +322,34 @@ Redis 只作为可丢弃的查询缓存。连接失败时应用回源数据库
|
||||
`allkeys-lfu` 淘汰策略，不启用持久化。`compose.app.example.yml` 不创建 Redis；
|
||||
如需连接外部 Redis，在 `.env` 中配置上述连接串即可。
|
||||
|
||||
### OpenTelemetry 与慢查询定位
|
||||
|
||||
应用已接入 OpenTelemetry 的 ASP.NET Core、HttpClient、.NET Runtime 指标，并通过
|
||||
`Jiaowu.Api.Database` ActivitySource 和 Meter 记录 EF Core 数据库命令。配置
|
||||
`OTEL_EXPORTER_OTLP_ENDPOINT` 后才启动 OpenTelemetry SDK 并向 OTLP Collector 外发；
|
||||
未配置时不会创建无处消费的请求 Span，也不会尝试连接本地 Collector，结构化慢查询日志
|
||||
仍然有效。
|
||||
|
||||
```text
|
||||
Observability__Enabled=true
|
||||
Observability__ServiceName=jiaowu-api
|
||||
Observability__SlowQueryThresholdMilliseconds=500
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector.example.edu.cn:4317
|
||||
```
|
||||
|
||||
数据库指标包括 `jiaowu.db.command.duration`、`jiaowu.db.command.slow` 和
|
||||
`jiaowu.db.command.failed`。为关键 EF 查询添加 `TagWith("模块.查询名")` 后，日志和
|
||||
追踪会直接显示该稳定名称；无标签查询只显示操作类型和 SQL 模板哈希。默认
|
||||
`Observability__IncludeSqlText=false`，不会把 SQL、参数值或连接串发送到日志和追踪
|
||||
系统。仅在受控诊断窗口内临时启用完整 SQL 模板，并限制 Collector 权限与保留时间。
|
||||
|
||||
应用侧阈值用于关联接口、TraceId 和查询名称；生产 MySQL 还应由数据库管理员启用慢查询
|
||||
日志，并将 `long_query_time` 设为与应用阈值一致。先按查询哈希/标签汇总高频慢查询，再
|
||||
对脱敏后的 `SELECT` 在测试库或只读副本执行 `EXPLAIN ANALYZE`，根据实际扫描行数和循环
|
||||
次数决定是否补组合索引或改写投影。`EXPLAIN ANALYZE` 会真实执行语句，不能直接用于生产
|
||||
写操作。参考 [MySQL 慢查询日志](https://dev.mysql.com/doc/refman/8.0/en/slow-query-log.html)
|
||||
和 [MySQL 8.4 EXPLAIN](https://dev.mysql.com/doc/refman/8.4/en/explain.html)。
|
||||
|
||||
### 后台任务与 RabbitMQ
|
||||
|
||||
自动排课、课表发布和补考自动生成使用数据库 Outbox 保存任务消息。创建业务任务与
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
using System.Data.Common;
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Microsoft.EntityFrameworkCore.Diagnostics;
|
||||
|
||||
namespace Jiaowu.Api.Infrastructure.Observability;
|
||||
|
||||
public sealed class DatabaseCommandTelemetryInterceptor(
|
||||
ObservabilityOptions options,
|
||||
ILogger<DatabaseCommandTelemetryInterceptor> logger)
|
||||
: DbCommandInterceptor
|
||||
{
|
||||
public const string ActivitySourceName = "Jiaowu.Api.Database";
|
||||
public const string MeterName = "Jiaowu.Api.Database";
|
||||
|
||||
private static readonly ActivitySource ActivitySource =
|
||||
new(ActivitySourceName);
|
||||
private static readonly Meter Meter = new(MeterName);
|
||||
private static readonly Histogram<double> CommandDuration =
|
||||
Meter.CreateHistogram<double>(
|
||||
"jiaowu.db.command.duration",
|
||||
"ms",
|
||||
"EF Core database command duration");
|
||||
private static readonly Counter<long> SlowCommandCount =
|
||||
Meter.CreateCounter<long>(
|
||||
"jiaowu.db.command.slow",
|
||||
"{command}",
|
||||
"EF Core commands exceeding the configured slow-query threshold");
|
||||
private static readonly Counter<long> FailedCommandCount =
|
||||
Meter.CreateCounter<long>(
|
||||
"jiaowu.db.command.failed",
|
||||
"{command}",
|
||||
"Failed EF Core database commands");
|
||||
|
||||
public override DbDataReader ReaderExecuted(
|
||||
DbCommand command,
|
||||
CommandExecutedEventData eventData,
|
||||
DbDataReader result)
|
||||
{
|
||||
Observe(command, eventData.Duration, "reader");
|
||||
return result;
|
||||
}
|
||||
|
||||
public override ValueTask<DbDataReader> ReaderExecutedAsync(
|
||||
DbCommand command,
|
||||
CommandExecutedEventData eventData,
|
||||
DbDataReader result,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
Observe(command, eventData.Duration, "reader");
|
||||
return ValueTask.FromResult(result);
|
||||
}
|
||||
|
||||
public override int NonQueryExecuted(
|
||||
DbCommand command,
|
||||
CommandExecutedEventData eventData,
|
||||
int result)
|
||||
{
|
||||
Observe(command, eventData.Duration, "nonquery");
|
||||
return result;
|
||||
}
|
||||
|
||||
public override ValueTask<int> NonQueryExecutedAsync(
|
||||
DbCommand command,
|
||||
CommandExecutedEventData eventData,
|
||||
int result,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
Observe(command, eventData.Duration, "nonquery");
|
||||
return ValueTask.FromResult(result);
|
||||
}
|
||||
|
||||
public override object? ScalarExecuted(
|
||||
DbCommand command,
|
||||
CommandExecutedEventData eventData,
|
||||
object? result)
|
||||
{
|
||||
Observe(command, eventData.Duration, "scalar");
|
||||
return result;
|
||||
}
|
||||
|
||||
public override ValueTask<object?> ScalarExecutedAsync(
|
||||
DbCommand command,
|
||||
CommandExecutedEventData eventData,
|
||||
object? result,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
Observe(command, eventData.Duration, "scalar");
|
||||
return ValueTask.FromResult(result);
|
||||
}
|
||||
|
||||
public override void CommandFailed(
|
||||
DbCommand command,
|
||||
CommandErrorEventData eventData) =>
|
||||
Observe(
|
||||
command,
|
||||
eventData.Duration,
|
||||
"failed",
|
||||
eventData.Exception.GetType().Name);
|
||||
|
||||
public override Task CommandFailedAsync(
|
||||
DbCommand command,
|
||||
CommandErrorEventData eventData,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
Observe(
|
||||
command,
|
||||
eventData.Duration,
|
||||
"failed",
|
||||
eventData.Exception.GetType().Name);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public override void CommandCanceled(
|
||||
DbCommand command,
|
||||
CommandEndEventData eventData) =>
|
||||
Observe(command, eventData.Duration, "canceled", "canceled");
|
||||
|
||||
public override Task CommandCanceledAsync(
|
||||
DbCommand command,
|
||||
CommandEndEventData eventData,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
Observe(command, eventData.Duration, "canceled", "canceled");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private void Observe(
|
||||
DbCommand command,
|
||||
TimeSpan duration,
|
||||
string commandKind,
|
||||
string? errorType = null)
|
||||
{
|
||||
if (!options.Enabled) return;
|
||||
|
||||
var queryName = GetQueryName(command.CommandText);
|
||||
var statementHash = GetStatementHash(command.CommandText);
|
||||
var provider = GetProviderName(command);
|
||||
var traceId = Activity.Current?.TraceId.ToString() ?? "none";
|
||||
var tags = new TagList
|
||||
{
|
||||
{ "db.system.name", provider },
|
||||
{ "db.operation.name", commandKind },
|
||||
{ "db.query.name", queryName }
|
||||
};
|
||||
if (errorType is not null)
|
||||
tags.Add("error.type", errorType);
|
||||
|
||||
var durationMilliseconds = duration.TotalMilliseconds;
|
||||
CommandDuration.Record(durationMilliseconds, tags);
|
||||
if (errorType is not null)
|
||||
FailedCommandCount.Add(1, tags);
|
||||
|
||||
using var activity = ActivitySource.StartActivity(
|
||||
ActivityKind.Client,
|
||||
Activity.Current?.Context ?? default,
|
||||
startTime: DateTimeOffset.UtcNow - duration,
|
||||
name: queryName);
|
||||
if (activity is not null)
|
||||
{
|
||||
activity.SetTag("db.system.name", provider);
|
||||
activity.SetTag("db.operation.name", commandKind);
|
||||
activity.SetTag("db.query.name", queryName);
|
||||
activity.SetTag("db.statement.hash", statementHash);
|
||||
activity.SetTag(
|
||||
"db.namespace",
|
||||
EmptyToNull(command.Connection?.Database));
|
||||
if (options.IncludeSqlText)
|
||||
{
|
||||
activity.SetTag(
|
||||
"db.query.text",
|
||||
Truncate(command.CommandText, options.MaximumSqlTextLength));
|
||||
}
|
||||
if (errorType is not null)
|
||||
{
|
||||
activity.SetTag("error.type", errorType);
|
||||
activity.SetStatus(ActivityStatusCode.Error, errorType);
|
||||
}
|
||||
activity.SetEndTime(DateTime.UtcNow);
|
||||
}
|
||||
|
||||
if (errorType is not null)
|
||||
{
|
||||
logger.LogError(
|
||||
"Database command failed after {DurationMs:F1} ms: " +
|
||||
"{QueryName} ({CommandKind}, {Provider}, hash {StatementHash}, " +
|
||||
"error {ErrorType}, trace {TraceId}).",
|
||||
durationMilliseconds,
|
||||
queryName,
|
||||
commandKind,
|
||||
provider,
|
||||
statementHash,
|
||||
errorType,
|
||||
traceId);
|
||||
return;
|
||||
}
|
||||
|
||||
if (durationMilliseconds < options.SlowQueryThresholdMilliseconds)
|
||||
return;
|
||||
|
||||
SlowCommandCount.Add(1, tags);
|
||||
if (options.IncludeSqlText)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Slow database command took {DurationMs:F1} ms: " +
|
||||
"{QueryName} ({CommandKind}, {Provider}, hash {StatementHash}, " +
|
||||
"trace {TraceId}). " +
|
||||
"SQL template: {SqlTemplate}",
|
||||
durationMilliseconds,
|
||||
queryName,
|
||||
commandKind,
|
||||
provider,
|
||||
statementHash,
|
||||
traceId,
|
||||
Truncate(command.CommandText, options.MaximumSqlTextLength));
|
||||
}
|
||||
else
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Slow database command took {DurationMs:F1} ms: " +
|
||||
"{QueryName} ({CommandKind}, {Provider}, hash {StatementHash}, " +
|
||||
"trace {TraceId}).",
|
||||
durationMilliseconds,
|
||||
queryName,
|
||||
commandKind,
|
||||
provider,
|
||||
statementHash,
|
||||
traceId);
|
||||
}
|
||||
}
|
||||
|
||||
internal static string GetQueryName(string commandText)
|
||||
{
|
||||
using var reader = new StringReader(commandText);
|
||||
while (reader.ReadLine() is { } line)
|
||||
{
|
||||
var trimmed = line.Trim();
|
||||
if (trimmed.Length == 0) continue;
|
||||
if (trimmed.StartsWith("-- ", StringComparison.Ordinal))
|
||||
return Truncate(trimmed[3..].Trim(), 120);
|
||||
return $"{FirstToken(trimmed)}:{GetStatementHash(commandText)}";
|
||||
}
|
||||
|
||||
return $"unknown:{GetStatementHash(commandText)}";
|
||||
}
|
||||
|
||||
internal static string GetStatementHash(string commandText)
|
||||
{
|
||||
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(commandText));
|
||||
return Convert.ToHexString(bytes.AsSpan(0, 6)).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string FirstToken(string value)
|
||||
{
|
||||
var end = value.IndexOfAny([' ', '\t', '\r', '\n', '(']);
|
||||
var token = end < 0 ? value : value[..end];
|
||||
return token.Length == 0
|
||||
? "command"
|
||||
: token.ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string GetProviderName(DbCommand command)
|
||||
{
|
||||
var typeName = command.GetType().FullName ?? command.GetType().Name;
|
||||
if (typeName.Contains("MySql", StringComparison.OrdinalIgnoreCase))
|
||||
return "mysql";
|
||||
if (typeName.Contains("Sqlite", StringComparison.OrdinalIgnoreCase))
|
||||
return "sqlite";
|
||||
return "other_sql";
|
||||
}
|
||||
|
||||
private static string? EmptyToNull(string? value) =>
|
||||
string.IsNullOrWhiteSpace(value) ? null : value;
|
||||
|
||||
private static string Truncate(string value, int maximumLength) =>
|
||||
value.Length <= maximumLength
|
||||
? value
|
||||
: value[..maximumLength];
|
||||
}
|
||||
Loaded 3 of 9 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user