.NET 6多请求下AWS Secrets Manager抛出HttpErrorResponseException

基于.NET 6编写的应用代码逻辑校验通过，初始运行状态正常，但当API累计接收约100次连接后，会固定抛出如下异常：

Amazon.SecretsManager.AmazonSecretsManagerException: The service returned an error. See inner exception for details. ---> Amazon.Runtime.Internal.HttpErrorResponseException: Exception of type 'Amazon.Runtime.Internal.HttpErrorResponseException' was thrown. at Amazon.Runtime.HttpWebRequestMessage.GetResponseAsync(CancellationToken cancellationToken) at Amazon.Runtime.Internal.HttpHandler1.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.Unmarshaller.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.ErrorHandler.InvokeAsync[T](IExecutionContext executionContext) --- End of inner exception stack trace --- at Amazon.Runtime.Internal.HttpErrorResponseExceptionHandler.HandleExceptionStream(IRequestContext requestContext, IWebResponseData httpErrorResponse, HttpErrorResponseException exception, Stream responseStream) at Amazon.Runtime.Internal.HttpErrorResponseExceptionHandler.HandleExceptionAsync(IExecutionContext executionContext, HttpErrorResponseException exception) at Amazon.Runtime.Internal.ExceptionHandler1.HandleAsync(IExecutionContext executionContext, Exception exception) at Amazon.Runtime.Internal.ErrorHandler.ProcessExceptionAsync(IExecutionContext executionContext, Exception exception) at Amazon.Runtime.Internal.ErrorHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.CallbackHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.EndpointDiscoveryHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.EndpointDiscoveryHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.CredentialsRetriever.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.RetryHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.RetryHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.CallbackHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.CallbackHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.ErrorCallbackHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.Runtime.Internal.MetricsHandler.InvokeAsync[T](IExecutionContext executionContext) at Amazon.SecretsManager.Extensions.Caching.SecretCacheItem.ExecuteRefreshAsync() at Amazon.SecretsManager.Extensions.Caching.SecretCacheObject1.RefreshAsync() at Amazon.SecretsManager.Extensions.Caching.SecretCacheObject1.GetSecretValue() at Amazon.SecretsManager.Extensions.Caching.SecretsManagerCache.GetSecretString(String secretId)

Amazon模块注册代码 #

public static class AmazonModule
{
    public static void AmazonServices(this IServiceCollection services, IConfiguration configuration)
    {
        
        services.AddSingleton<IAmazonSecretsManager>(config => new AmazonSecretsManagerClient(RegionEndpoint.GetBySystemName(_section[Constants.AWS_SECRETMANAGER_REGION])));
    
        var assemblyToScan = Assembly.GetAssembly(typeof(SecretsManagerService));

        services.RegisterAssemblyPublicNonGenericClasses(assemblyToScan)
            .Where(t => t.Name.EndsWith("Service"))
            .AsPublicImplementedInterfaces(ServiceLifetime.Scoped);
      
        services.AddCognitoIdentity();
    }
}

数据库模块注册代码 #

public static class DatabaseModule
{
    public static void RegisterDatabaseModule(this IServiceCollection services, IConfiguration configuration)
    {
        services.AddDbContext<BoletoPixDataContext>((ctx, options) =>
        {
            string connection = "";
            using (IServiceScope scope = ctx.GetRequiredService<IServiceScopeFactory>().CreateScope())
            {
                var secretsManagerService = ctx.GetService<ISecretsManagerService>();
                connection = secretsManagerService.GetSecret<ConnectionString>(configuration[Constants.AWS_CONFIG_SECRETMANAGER_SECTION + ":" + Constants.AWS_SECRETMANAGER_DATABASE]);
            }

            options.UseSqlServer(connection);


            var auditNet = Convert.ToBoolean(configuration.GetSection("AuditNetEntity").Value);
            if (auditNet)
            {
                Configuration
                    .Setup()
                    .ForAnyContext(config => config
                        .IncludeEntityObjects()
                        .AuditEventType("{database}_{context}"))
                    .UseOptOut();
                options.AddInterceptors(new AuditSaveChangesInterceptor());
            }
        });
    }
}

SecretsManager服务实现代码 #

public class SecretsManagerService : ISecretsManagerService
{
    // 省略部分冗余代码
    public SecretsManagerService(IConfiguration configuration, Func<object, LoggerConfig> loggerConfig, IAmazonSecretsManager amazonSecretsManager)
    {
        _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
        var section = _configuration.GetSection(Constants.AWS_CONFIG_SECRETMANAGER_SECTION);
        _loggerConfig = loggerConfig(this) ?? throw new ArgumentNullException(nameof(loggerConfig));
        _client = amazonSecretsManager ?? throw new ArgumentNullException(nameof(amazonSecretsManager));
        cache = new SecretsManagerCache(_client);
    }
    
    public string GetSecret<T>(string secretName) where T : IConfigs
    {
        var cachedString = GetCachedSecret(secretName);
        if (string.IsNullOrEmpty(cachedString))
        {
            var request = new GetSecretValueRequest()
            {
                SecretId = secretName,
                VersionStage = "AWSCURRENT"
            };

            try
            {
                LoggerSingleton.Info(_loggerConfig, "Local Request");
                var response = _client.GetSecretValueAsync(request).Result;
                
                var connection = JsonConvert.DeserializeObject<T>(response.SecretString);
                return connection?.GetString() ?? "";
            }
            catch (Exception ex)
            {
                LoggerSingleton.Info(_loggerConfig, "Error on AWS SECRET " + ex.Message);
                throw;
            }
        }
        else
        {
            var connection = JsonConvert.DeserializeObject<ConnectionString>(cachedString);
            return connection?.GetString() ?? "";
        }
    }
    
    public string GetCachedSecret(string secretName)
    {
        if (cache != null)
        {
            var mySecret = cache.GetSecretString(secretName).Result;
            return mySecret;
        }
        return "";
    }
}

Program.cs服务注册入口 #

builder.Services.AmazonServices(configuration);
builder.Services.RegisterDatabaseModule(configuration);

• 同步阻塞异步调用导致线程池耗尽。代码中两处直接调用.Result阻塞异步方法：_client.GetSecretValueAsync(request).Result、cache.GetSecretString(secretName).Result。ASP.NET Core虽然没有传统.NET Framework的同步上下文死锁问题，但同步阻塞异步IO会持续占用工作线程等待响应，当并发量上升时，线程池可用线程被耗尽，无法处理Secrets Manager HTTP请求的IO完成回调，最终请求超时抛出异常，这也是故障固定在累计约100次请求时触发的直接原因——线程池注入新线程的速度约为每秒1个，当阻塞速度超过线程注入速度时服务就会雪崩。
• DbContext配置逻辑生命周期错误。AddDbContext传入的配置委托会在每次创建DbContext实例时执行，而非应用启动时仅执行一次，也就是说每次请求创建DbContext都会调用Secrets Manager拉取密钥，缓存完全没有起到预期作用，产生了大量不必要的HTTP请求，进一步放大了线程阻塞的影响。
• 缓存实例创建方式不合理。SecretsManagerCache在Scoped生命周期的SecretsManagerService构造函数中初始化，每次请求都会生成新的缓存实例，完全丧失了缓存的全局复用能力，额外增加了不必要的API调用开销。
• 代码存在显性编译错误：AmazonModule中初始化AmazonSecretsManagerClient时使用的_section变量未定义、未赋值，直接运行会抛出编译异常。

• 全链路使用异步调用，禁止在异步API上调用.Result、.Wait()等同步阻塞方法。将GetSecret、GetCachedSecret改造为异步方法，全程使用await关键字调用异步接口，释放工作线程，避免线程池耗尽。
• 将数据库连接字符串获取逻辑移到应用启动阶段，仅执行一次。启动时从Secrets Manager拉取到连接字符串后，直接传入UseSqlServer方法，不要放在AddDbContext的配置委托中重复执行。
• 将SecretsManagerCache注册为单例服务，全局复用同一个缓存实例，配置合理的缓存TTL（比如1小时），减少不必要的Secrets Manager API调用。
• 修复AmazonModule中的未定义变量问题，从传入的configuration对象中读取AWS区域配置后，再初始化AmazonSecretsManagerClient。

内容的提问来源于stack exchange，提问作者Wesley Reuel Marques Silva