接入 OpenTelemetry 1.17.0,覆盖 HTTP、HttpClient、.NET Runtime 和数据库链路。
新增 EF Core 数据库拦截器,记录耗时、失败数、慢查询数、TraceId、查询标签和 SQL 哈希;默认不记录完整 SQL及参数。 未配置 OTLP Collector 时不启动 SDK,避免无收益的性能开销。 为课表的作息、课程、灵活课程、考试、实验等六类查询增加稳定标签。 增加可配置的 500ms 慢查询阈值,以及生产环境变量示例。 README 补充 MySQL 慢查询与 EXPLAIN ANALYZE 操作规范。
This commit is contained in:
1 parent
a369b4fe02
commit
ca7148ab61
9 files changed
+489
-1
No files matched your search
@@ -36,6 +36,16 @@ Cache__AnalyticsExpirationMinutes=3
|
|||||||
Cache__AnalyticsLocalExpirationSeconds=30
|
Cache__AnalyticsLocalExpirationSeconds=30
|
||||||
Cache__MaximumPayloadKilobytes=2048
|
Cache__MaximumPayloadKilobytes=2048
|
||||||
|
|
||||||
|
# OpenTelemetry 默认收集 HTTP、运行时和数据库指标;配置 OTLP 地址后才会外发。
|
||||||
|
Observability__Enabled=true
|
||||||
|
Observability__ServiceName=jiaowu-api
|
||||||
|
Observability__SlowQueryThresholdMilliseconds=500
|
||||||
|
# 默认不记录完整 SQL,避免日志或追踪系统接触业务数据。
|
||||||
|
Observability__IncludeSqlText=false
|
||||||
|
Observability__MaximumSqlTextLength=2000
|
||||||
|
# OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector.example.edu.cn:4317
|
||||||
|
# OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer%20REPLACE_WITH_TOKEN
|
||||||
|
|
||||||
# 运维控制台备份目录必须位于持久化、仅服务账号可写的位置。
|
# 运维控制台备份目录必须位于持久化、仅服务账号可写的位置。
|
||||||
Operations__BackupDirectory=/var/lib/jiaowu/backups
|
Operations__BackupDirectory=/var/lib/jiaowu/backups
|
||||||
Operations__BackupWarningHours=24
|
Operations__BackupWarningHours=24
|
||||||
|
|||||||
@@ -322,6 +322,34 @@ Redis 只作为可丢弃的查询缓存。连接失败时应用回源数据库
|
|||||||
`allkeys-lfu` 淘汰策略,不启用持久化。`compose.app.example.yml` 不创建 Redis;
|
`allkeys-lfu` 淘汰策略,不启用持久化。`compose.app.example.yml` 不创建 Redis;
|
||||||
如需连接外部 Redis,在 `.env` 中配置上述连接串即可。
|
如需连接外部 Redis,在 `.env` 中配置上述连接串即可。
|
||||||
|
|
||||||
|
### OpenTelemetry 与慢查询定位
|
||||||
|
|
||||||
|
应用已接入 OpenTelemetry 的 ASP.NET Core、HttpClient、.NET Runtime 指标,并通过
|
||||||
|
`Jiaowu.Api.Database` ActivitySource 和 Meter 记录 EF Core 数据库命令。配置
|
||||||
|
`OTEL_EXPORTER_OTLP_ENDPOINT` 后才启动 OpenTelemetry SDK 并向 OTLP Collector 外发;
|
||||||
|
未配置时不会创建无处消费的请求 Span,也不会尝试连接本地 Collector,结构化慢查询日志
|
||||||
|
仍然有效。
|
||||||
|
|
||||||
|
```text
|
||||||
|
Observability__Enabled=true
|
||||||
|
Observability__ServiceName=jiaowu-api
|
||||||
|
Observability__SlowQueryThresholdMilliseconds=500
|
||||||
|
OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector.example.edu.cn:4317
|
||||||
|
```
|
||||||
|
|
||||||
|
数据库指标包括 `jiaowu.db.command.duration`、`jiaowu.db.command.slow` 和
|
||||||
|
`jiaowu.db.command.failed`。为关键 EF 查询添加 `TagWith("模块.查询名")` 后,日志和
|
||||||
|
追踪会直接显示该稳定名称;无标签查询只显示操作类型和 SQL 模板哈希。默认
|
||||||
|
`Observability__IncludeSqlText=false`,不会把 SQL、参数值或连接串发送到日志和追踪
|
||||||
|
系统。仅在受控诊断窗口内临时启用完整 SQL 模板,并限制 Collector 权限与保留时间。
|
||||||
|
|
||||||
|
应用侧阈值用于关联接口、TraceId 和查询名称;生产 MySQL 还应由数据库管理员启用慢查询
|
||||||
|
日志,并将 `long_query_time` 设为与应用阈值一致。先按查询哈希/标签汇总高频慢查询,再
|
||||||
|
对脱敏后的 `SELECT` 在测试库或只读副本执行 `EXPLAIN ANALYZE`,根据实际扫描行数和循环
|
||||||
|
次数决定是否补组合索引或改写投影。`EXPLAIN ANALYZE` 会真实执行语句,不能直接用于生产
|
||||||
|
写操作。参考 [MySQL 慢查询日志](https://dev.mysql.com/doc/refman/8.0/en/slow-query-log.html)
|
||||||
|
和 [MySQL 8.4 EXPLAIN](https://dev.mysql.com/doc/refman/8.4/en/explain.html)。
|
||||||
|
|
||||||
### 后台任务与 RabbitMQ
|
### 后台任务与 RabbitMQ
|
||||||
|
|
||||||
自动排课、课表发布和补考自动生成使用数据库 Outbox 保存任务消息。创建业务任务与
|
自动排课、课表发布和补考自动生成使用数据库 Outbox 保存任务消息。创建业务任务与
|
||||||
|
|||||||
@@ -0,0 +1,281 @@
|
|||||||
|
using System.Data.Common;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using System.Diagnostics.Metrics;
|
||||||
|
using System.Security.Cryptography;
|
||||||
|
using System.Text;
|
||||||
|
using Microsoft.EntityFrameworkCore.Diagnostics;
|
||||||
|
|
||||||
|
namespace Jiaowu.Api.Infrastructure.Observability;
|
||||||
|
|
||||||
|
public sealed class DatabaseCommandTelemetryInterceptor(
|
||||||
|
ObservabilityOptions options,
|
||||||
|
ILogger<DatabaseCommandTelemetryInterceptor> logger)
|
||||||
|
: DbCommandInterceptor
|
||||||
|
{
|
||||||
|
public const string ActivitySourceName = "Jiaowu.Api.Database";
|
||||||
|
public const string MeterName = "Jiaowu.Api.Database";
|
||||||
|
|
||||||
|
private static readonly ActivitySource ActivitySource =
|
||||||
|
new(ActivitySourceName);
|
||||||
|
private static readonly Meter Meter = new(MeterName);
|
||||||
|
private static readonly Histogram<double> CommandDuration =
|
||||||
|
Meter.CreateHistogram<double>(
|
||||||
|
"jiaowu.db.command.duration",
|
||||||
|
"ms",
|
||||||
|
"EF Core database command duration");
|
||||||
|
private static readonly Counter<long> SlowCommandCount =
|
||||||
|
Meter.CreateCounter<long>(
|
||||||
|
"jiaowu.db.command.slow",
|
||||||
|
"{command}",
|
||||||
|
"EF Core commands exceeding the configured slow-query threshold");
|
||||||
|
private static readonly Counter<long> FailedCommandCount =
|
||||||
|
Meter.CreateCounter<long>(
|
||||||
|
"jiaowu.db.command.failed",
|
||||||
|
"{command}",
|
||||||
|
"Failed EF Core database commands");
|
||||||
|
|
||||||
|
public override DbDataReader ReaderExecuted(
|
||||||
|
DbCommand command,
|
||||||
|
CommandExecutedEventData eventData,
|
||||||
|
DbDataReader result)
|
||||||
|
{
|
||||||
|
Observe(command, eventData.Duration, "reader");
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override ValueTask<DbDataReader> ReaderExecutedAsync(
|
||||||
|
DbCommand command,
|
||||||
|
CommandExecutedEventData eventData,
|
||||||
|
DbDataReader result,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
Observe(command, eventData.Duration, "reader");
|
||||||
|
return ValueTask.FromResult(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
public override int NonQueryExecuted(
|
||||||
|
DbCommand command,
|
||||||
|
CommandExecutedEventData eventData,
|
||||||
|
int result)
|
||||||
|
{
|
||||||
|
Observe(command, eventData.Duration, "nonquery");
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override ValueTask<int> NonQueryExecutedAsync(
|
||||||
|
DbCommand command,
|
||||||
|
CommandExecutedEventData eventData,
|
||||||
|
int result,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
Observe(command, eventData.Duration, "nonquery");
|
||||||
|
return ValueTask.FromResult(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
public override object? ScalarExecuted(
|
||||||
|
DbCommand command,
|
||||||
|
CommandExecutedEventData eventData,
|
||||||
|
object? result)
|
||||||
|
{
|
||||||
|
Observe(command, eventData.Duration, "scalar");
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override ValueTask<object?> ScalarExecutedAsync(
|
||||||
|
DbCommand command,
|
||||||
|
CommandExecutedEventData eventData,
|
||||||
|
object? result,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
Observe(command, eventData.Duration, "scalar");
|
||||||
|
return ValueTask.FromResult(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void CommandFailed(
|
||||||
|
DbCommand command,
|
||||||
|
CommandErrorEventData eventData) =>
|
||||||
|
Observe(
|
||||||
|
command,
|
||||||
|
eventData.Duration,
|
||||||
|
"failed",
|
||||||
|
eventData.Exception.GetType().Name);
|
||||||
|
|
||||||
|
public override Task CommandFailedAsync(
|
||||||
|
DbCommand command,
|
||||||
|
CommandErrorEventData eventData,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
Observe(
|
||||||
|
command,
|
||||||
|
eventData.Duration,
|
||||||
|
"failed",
|
||||||
|
eventData.Exception.GetType().Name);
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void CommandCanceled(
|
||||||
|
DbCommand command,
|
||||||
|
CommandEndEventData eventData) =>
|
||||||
|
Observe(command, eventData.Duration, "canceled", "canceled");
|
||||||
|
|
||||||
|
public override Task CommandCanceledAsync(
|
||||||
|
DbCommand command,
|
||||||
|
CommandEndEventData eventData,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
Observe(command, eventData.Duration, "canceled", "canceled");
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void Observe(
|
||||||
|
DbCommand command,
|
||||||
|
TimeSpan duration,
|
||||||
|
string commandKind,
|
||||||
|
string? errorType = null)
|
||||||
|
{
|
||||||
|
if (!options.Enabled) return;
|
||||||
|
|
||||||
|
var queryName = GetQueryName(command.CommandText);
|
||||||
|
var statementHash = GetStatementHash(command.CommandText);
|
||||||
|
var provider = GetProviderName(command);
|
||||||
|
var traceId = Activity.Current?.TraceId.ToString() ?? "none";
|
||||||
|
var tags = new TagList
|
||||||
|
{
|
||||||
|
{ "db.system.name", provider },
|
||||||
|
{ "db.operation.name", commandKind },
|
||||||
|
{ "db.query.name", queryName }
|
||||||
|
};
|
||||||
|
if (errorType is not null)
|
||||||
|
tags.Add("error.type", errorType);
|
||||||
|
|
||||||
|
var durationMilliseconds = duration.TotalMilliseconds;
|
||||||
|
CommandDuration.Record(durationMilliseconds, tags);
|
||||||
|
if (errorType is not null)
|
||||||
|
FailedCommandCount.Add(1, tags);
|
||||||
|
|
||||||
|
using var activity = ActivitySource.StartActivity(
|
||||||
|
ActivityKind.Client,
|
||||||
|
Activity.Current?.Context ?? default,
|
||||||
|
startTime: DateTimeOffset.UtcNow - duration,
|
||||||
|
name: queryName);
|
||||||
|
if (activity is not null)
|
||||||
|
{
|
||||||
|
activity.SetTag("db.system.name", provider);
|
||||||
|
activity.SetTag("db.operation.name", commandKind);
|
||||||
|
activity.SetTag("db.query.name", queryName);
|
||||||
|
activity.SetTag("db.statement.hash", statementHash);
|
||||||
|
activity.SetTag(
|
||||||
|
"db.namespace",
|
||||||
|
EmptyToNull(command.Connection?.Database));
|
||||||
|
if (options.IncludeSqlText)
|
||||||
|
{
|
||||||
|
activity.SetTag(
|
||||||
|
"db.query.text",
|
||||||
|
Truncate(command.CommandText, options.MaximumSqlTextLength));
|
||||||
|
}
|
||||||
|
if (errorType is not null)
|
||||||
|
{
|
||||||
|
activity.SetTag("error.type", errorType);
|
||||||
|
activity.SetStatus(ActivityStatusCode.Error, errorType);
|
||||||
|
}
|
||||||
|
activity.SetEndTime(DateTime.UtcNow);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (errorType is not null)
|
||||||
|
{
|
||||||
|
logger.LogError(
|
||||||
|
"Database command failed after {DurationMs:F1} ms: " +
|
||||||
|
"{QueryName} ({CommandKind}, {Provider}, hash {StatementHash}, " +
|
||||||
|
"error {ErrorType}, trace {TraceId}).",
|
||||||
|
durationMilliseconds,
|
||||||
|
queryName,
|
||||||
|
commandKind,
|
||||||
|
provider,
|
||||||
|
statementHash,
|
||||||
|
errorType,
|
||||||
|
traceId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (durationMilliseconds < options.SlowQueryThresholdMilliseconds)
|
||||||
|
return;
|
||||||
|
|
||||||
|
SlowCommandCount.Add(1, tags);
|
||||||
|
if (options.IncludeSqlText)
|
||||||
|
{
|
||||||
|
logger.LogWarning(
|
||||||
|
"Slow database command took {DurationMs:F1} ms: " +
|
||||||
|
"{QueryName} ({CommandKind}, {Provider}, hash {StatementHash}, " +
|
||||||
|
"trace {TraceId}). " +
|
||||||
|
"SQL template: {SqlTemplate}",
|
||||||
|
durationMilliseconds,
|
||||||
|
queryName,
|
||||||
|
commandKind,
|
||||||
|
provider,
|
||||||
|
statementHash,
|
||||||
|
traceId,
|
||||||
|
Truncate(command.CommandText, options.MaximumSqlTextLength));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
logger.LogWarning(
|
||||||
|
"Slow database command took {DurationMs:F1} ms: " +
|
||||||
|
"{QueryName} ({CommandKind}, {Provider}, hash {StatementHash}, " +
|
||||||
|
"trace {TraceId}).",
|
||||||
|
durationMilliseconds,
|
||||||
|
queryName,
|
||||||
|
commandKind,
|
||||||
|
provider,
|
||||||
|
statementHash,
|
||||||
|
traceId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static string GetQueryName(string commandText)
|
||||||
|
{
|
||||||
|
using var reader = new StringReader(commandText);
|
||||||
|
while (reader.ReadLine() is { } line)
|
||||||
|
{
|
||||||
|
var trimmed = line.Trim();
|
||||||
|
if (trimmed.Length == 0) continue;
|
||||||
|
if (trimmed.StartsWith("-- ", StringComparison.Ordinal))
|
||||||
|
return Truncate(trimmed[3..].Trim(), 120);
|
||||||
|
return $"{FirstToken(trimmed)}:{GetStatementHash(commandText)}";
|
||||||
|
}
|
||||||
|
|
||||||
|
return $"unknown:{GetStatementHash(commandText)}";
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static string GetStatementHash(string commandText)
|
||||||
|
{
|
||||||
|
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(commandText));
|
||||||
|
return Convert.ToHexString(bytes.AsSpan(0, 6)).ToLowerInvariant();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string FirstToken(string value)
|
||||||
|
{
|
||||||
|
var end = value.IndexOfAny([' ', '\t', '\r', '\n', '(']);
|
||||||
|
var token = end < 0 ? value : value[..end];
|
||||||
|
return token.Length == 0
|
||||||
|
? "command"
|
||||||
|
: token.ToLowerInvariant();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string GetProviderName(DbCommand command)
|
||||||
|
{
|
||||||
|
var typeName = command.GetType().FullName ?? command.GetType().Name;
|
||||||
|
if (typeName.Contains("MySql", StringComparison.OrdinalIgnoreCase))
|
||||||
|
return "mysql";
|
||||||
|
if (typeName.Contains("Sqlite", StringComparison.OrdinalIgnoreCase))
|
||||||
|
return "sqlite";
|
||||||
|
return "other_sql";
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string? EmptyToNull(string? value) =>
|
||||||
|
string.IsNullOrWhiteSpace(value) ? null : value;
|
||||||
|
|
||||||
|
private static string Truncate(string value, int maximumLength) =>
|
||||||
|
value.Length <= maximumLength
|
||||||
|
? value
|
||||||
|
: value[..maximumLength];
|
||||||
|
}
|
||||||
Loaded 3 of 9 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user