ASP.NET Core 8 Audit.NET存Azure表报错，求解决方案

问题背景 #

在ASP.NET Core 8 Web API中基于Audit.NET实现审计追踪，使用AzureStorageTableDataProvider存储审计事件。DEV、QA等低环境运行正常，但部署到PRODUCTION环境时触发Bad Request错误：

The property value exceeds the maximum allowed size (64KB). If the property value is a string, it is UTF-16 encoded and the maximum number of characters should be 32K or less.
RequestId:xxxxx-yyyy-xxxx-yyyy-xxxxxxxx
Time:2025-01-16T04:59:02.9744140Z
Status: 400 (Bad Request)
ErrorCode: PropertyValueTooLarge

现有代码实现 #

审计Middleware配置 #

public void AuditSetupMiddleware(IApplicationBuilder app)
{
    // Add the audit Middleware to the pipeline
    app.UseAuditMiddleware(_ => _
        .FilterByRequest(r => !r.Path.Value!.EndsWith("favicon.ico") && !r.Method.Equals(nameof(HttpMethod.Get), StringComparison.OrdinalIgnoreCase))
        .WithEventType("{verb}:{url}")
        .IncludeHeaders()
        .IncludeResponseHeaders()
        .IncludeResponseBody());
}

审计输出存储配置 #

public void AuditSetupOutput(IApplicationBuilder app)
{
    var options = new JsonSerializerOptions()
    {
        WriteIndented = true
    };

    Configuration.Setup()
    .JsonSystemAdapter(options)
    .UseAzureTableStorage(config => config
        .Endpoint(new Uri(_appOptions.Value.AuditTrailStorageAccountUrl?.Replace(Constants.Constant.Blob, Constants.Constant.Table)!), new ManagedIdentityCredential(_appOptions.Value.UserAssignedClientId))
        .TableName(evt => $"{_purgeEntitiesOptions.Value.TargetTableName}{DateTime.UtcNow:MMMyyyy}")
        .ClientOptions(new TableClientOptions() { Retry = { MaxRetries = 3 } })
        .EntityBuilder(builder => builder
            .PartitionKey(auditEvent => auditEvent.Environment.UserName)
            .RowKey(auditEvent => Guid.NewGuid().ToString("N"))
            .Columns(col => col
                .FromDictionary(auditEvent => new Dictionary<string, object>()
                {
                    { "EventType", auditEvent.EventType },
                    { "UserName", auditEvent.Environment.UserName },
                    { "EventDuration", auditEvent.Duration },
                    { "DataSize", auditEvent.ToJson().Length },
                    { "Data", auditEvent.ToJson().Length >= 32000 ? CompressAuditEventData(auditEvent.ToJson()): auditEvent.ToJson()}
                }))));


    // Include the trace identifier in the audit events
    var httpContextAccessor = app.ApplicationServices.GetRequiredService<IHttpContextAccessor>();
    Configuration.AddCustomAction(ActionType.OnScopeCreated, scope =>
    {
        scope.SetCustomField("TraceId", httpContextAccessor.HttpContext?.TraceIdentifier);
    });

}

压缩方法 #

/// <summary>
/// Compress the AuditTrail data in case characters count is about 32 K or less to fix the error produced 
/// as part of Azure Table Storage limitation: The property value exceeds the maximum allowed size (64KB). 
/// If the property value is a string, it is UTF-16 encoded and the maximum number of characters should be 32K or less.
/// </summary>
/// <param name="auditTrailUnCompressedData"></param>
/// <returns></returns>
/// <exception cref="ArgumentNullException"></exception>
private static string CompressAuditEventData(string auditTrailUnCompressedData)
{
    if (string.IsNullOrWhiteSpace(auditTrailUnCompressedData))
    {
        throw new ArgumentNullException(nameof(auditTrailUnCompressedData));
    }

    byte[] dataToCompress = Encoding.UTF8.GetBytes(auditTrailUnCompressedData);
    byte[] compressedData = Compress(dataToCompress);
    return Encoding.UTF8.GetString(compressedData);
}

Audit过滤器配置 #

internal static MvcOptions AuditSetupFilter(this MvcOptions mvcOptions)
{
    mvcOptions.AddAuditFilter(config => config
        .LogRequestIf(r => !(r.Method.Equals(nameof(HttpMethod.Get), StringComparison.OrdinalIgnoreCase)) && !((r.HttpContext.Request.HasFormContentType)))
        .WithEventType("{verb} {controller}.{action}")
        .IncludeHeaders(ctx => !ctx.ModelState.IsValid)
        .IncludeModelState()
        .IncludeResponseHeaders()
        .SerializeActionParameters()
    );

    // Ignore ActionParameters
    Audit.Core.Configuration.AddCustomAction(ActionType.OnEventSaving, scope =>
    {
        scope.GetWebApiAuditAction().ActionParameters = null;
    });

    return mvcOptions;
}

Program.cs调用 #

builder.Services.AddControllers(config =>
{
    config.AuditSetupFilter();  
});

问题分析与修改方案 #

现有压缩方案存在两个核心问题：

1. 直接将压缩后的字节数组转UTF8字符串，可能引入无效字符，且无法保证压缩后的字符串长度符合Azure Table的限制
2. 阈值判断使用32000字符，而Azure Table的字符串限制是UTF-16编码下32768字符（对应64KB），阈值不准确

此外，不必要的JSON格式化、过大的响应体/参数收集也会增加Audit事件的体积。

修改后的代码示例 #

1. 修正压缩方法（使用Base64编码）

private static string CompressAuditEventData(string auditTrailUnCompressedData)
{
    if (string.IsNullOrWhiteSpace(auditTrailUnCompressedData))
    {
        throw new ArgumentNullException(nameof(auditTrailUnCompressedData));
    }

    byte[] dataToCompress = Encoding.UTF8.GetBytes(auditTrailUnCompressedData);
    using var memoryStream = new MemoryStream();
    using var gzipStream = new GZipStream(memoryStream, CompressionLevel.Optimal);
    gzipStream.Write(dataToCompress, 0, dataToCompress.Length);
    gzipStream.Close();
    // 将压缩后的字节数组转为Base64字符串，避免无效字符且长度可控
    return Convert.ToBase64String(memoryStream.ToArray());
}

// 读取审计数据时的解压方法（可选）
private static string DecompressAuditEventData(string compressedBase64Data)
{
    if (string.IsNullOrWhiteSpace(compressedBase64Data))
    {
        throw new ArgumentNullException(nameof(compressedBase64Data));
    }

    byte[] compressedData = Convert.FromBase64String(compressedBase64Data);
    using var memoryStream = new MemoryStream(compressedData);
    using var gzipStream = new GZipStream(memoryStream, CompressionMode.Decompress);
    using var resultStream = new MemoryStream();
    gzipStream.CopyTo(resultStream);
    return Encoding.UTF8.GetString(resultStream.ToArray());
}

2. 调整存储配置的阈值与字段

public void AuditSetupOutput(IApplicationBuilder app)
{
    var options = new JsonSerializerOptions()
    {
        WriteIndented = false // 关闭JSON缩进，减少字符串体积
    };

    Configuration.Setup()
    .JsonSystemAdapter(options)
    .UseAzureTableStorage(config => config
        .Endpoint(new Uri(_appOptions.Value.AuditTrailStorageAccountUrl?.Replace(Constants.Constant.Blob, Constants.Constant.Table)!), new ManagedIdentityCredential(_appOptions.Value.UserAssignedClientId))
        .TableName(evt => $"{_purgeEntitiesOptions.Value.TargetTableName}{DateTime.UtcNow:MMMyyyy}")
        .ClientOptions(new TableClientOptions() { Retry = { MaxRetries = 3 } })
        .EntityBuilder(builder => builder
            .PartitionKey(auditEvent => auditEvent.Environment.UserName)
            .RowKey(auditEvent => Guid.NewGuid().ToString("N"))
            .Columns(col => col
                .FromDictionary(auditEvent => 
                {
                    var auditJson = auditEvent.ToJson();
                    // 使用准确的32768字符阈值（UTF-16限制）
                    var isCompressed = auditJson.Length >= 32768;
                    var dataValue = isCompressed ? CompressAuditEventData(auditJson) : auditJson;
                    
                    return new Dictionary<string, object>()
                    {
                        { "EventType", auditEvent.EventType },
                        { "UserName", auditEvent.Environment.UserName },
                        { "EventDuration", auditEvent.Duration },
                        { "DataSize", auditJson.Length },
                        { "IsCompressed", isCompressed }, // 添加标记，方便后续读取时判断是否需要解压
                        { "Data", dataValue }
                    };
                }))));


    var httpContextAccessor = app.ApplicationServices.GetRequiredService<IHttpContextAccessor>();
    Configuration.AddCustomAction(ActionType.OnScopeCreated, scope =>
    {
        scope.SetCustomField("TraceId", httpContextAccessor.HttpContext?.TraceIdentifier);
    });

}

3. 优化审计数据收集范围

// 调整Middleware，限制响应体收集条件
public void AuditSetupMiddleware(IApplicationBuilder app)
{
    app.UseAuditMiddleware(_ => _
        .FilterByRequest(r => !r.Path.Value!.EndsWith("favicon.ico") && !r.Method.Equals(nameof(HttpMethod.Get), StringComparison.OrdinalIgnoreCase))
        .WithEventType("{verb}:{url}")
        .IncludeHeaders()
        .IncludeResponseHeaders()
        // 仅收集非成功状态或小体积的响应体
        .IncludeResponseBody(ctx => 
        {
            var response = ctx.HttpContext.Response;
            return response.StatusCode >= 400 || (response.ContentLength.HasValue && response.ContentLength.Value <= 30 * 1024);
        }));
}

// 调整Audit过滤器，减少不必要的数据收集
internal static MvcOptions AuditSetupFilter(this MvcOptions mvcOptions)
{
    mvcOptions.AddAuditFilter(config => config
        .LogRequestIf(r => !(r.Method.Equals(nameof(HttpMethod.Get), StringComparison.OrdinalIgnoreCase)) && !r.HttpContext.Request.HasFormContentType)
        .WithEventType("{verb} {controller}.{action}")
        .IncludeHeaders(ctx => !ctx.ModelState.IsValid)
        .IncludeModelState(ctx => !ctx.ModelState.IsValid) // 仅在ModelState无效时收集
        .IncludeResponseHeaders()
        .SerializeActionParameters(parameters => 
        {
            // 过滤掉大体积参数（如文件、字节数组）
            return parameters.Where(p => p.Value is not IFormFile && p.Value is not byte[]);
        })
    );

    Audit.Core.Configuration.AddCustomAction(ActionType.OnEventSaving, scope =>
    {
        scope.GetWebApiAuditAction().ActionParameters = null;
    });

    return mvcOptions;
}

内容的提问来源于stack exchange，提问作者santosh kumar patro