ASP.NET Core后台服务处理Azure队列消息时执行到数据库更新后无后续操作的诡异问题排查求助

我现在遇到一个超级头大的诡异问题：我的ASP.NET Core后台服务从Azure存储队列读取消息（是Dequeue不是Peek），调用外部API获取数据（API可能返回204，之后会重试），拿到有效数据后调用UpdateWorkerReportOfInjuryMetaData更新数据库的两个字段，最后要发布消息到另一个队列交给其他后台服务处理。

但奇怪的是，这个服务间歇性抽风：有些消息全程正常，可部分消息执行到数据库更新那一步之后，后面的所有操作就彻底没动静了——连Azure日志都没有任何输出，完全找不到排查的线索，就像程序在那之后静默“消失”了一样，实在搞不懂哪里出了问题，求各位大佬给点排查方向！

以下是我的后台服务核心代码：

public class ClaimRetrievalBackgroundService : BackgroundService 
{ 
    private readonly ILogger<ClaimRetrievalBackgroundService> _logger; 
    private readonly IServiceProvider _serviceProvider; 
    private readonly ISerializer<ClaimRetreivalQueueMessage> _serializer; 
    private readonly ICmsReportInjuryClient _cmsReportInjuryClient; 
    private readonly TimeSpan _queueCallDelayInSeconds = TimeSpan.FromSeconds(2); 

    public ClaimRetrievalBackgroundService(ILogger<ClaimRetrievalBackgroundService> logger, IServiceProvider serviceProvider, ISerializer<ClaimRetreivalQueueMessage> serializer, ICmsReportInjuryClient cmsReportInjuryClient) 
    { 
        _logger = logger; 
        _serviceProvider = serviceProvider; 
        _serializer = serializer; 
        _cmsReportInjuryClient = cmsReportInjuryClient; 
    } 

    protected override async Task ExecuteAsync(CancellationToken stoppingToken) 
    { 
        _logger.LogInformation("Starting the claim retrieval and cnn from CMS background service"); 
        await LookForClaimRetrievalRequestsAsync(stoppingToken); 
    } 

    public async Task LookForClaimRetrievalRequestsAsync(CancellationToken cancellationToken, bool isUnitTest = false) 
    { 
        while (!cancellationToken.IsCancellationRequested) 
        { 
            try 
            { 
                await ProcessNextClaimRetrievalRequestAsync(cancellationToken); 
            } 
            catch (Exception ex) 
            { 
                _logger.LogError(ex, "Unknown error occurred in LookForClaimRetrievalRequestsAsync"); 
            } 
            await Task.Delay(_queueCallDelayInSeconds, cancellationToken); 
        } 
    } 

    private async Task ProcessNextClaimRetrievalRequestAsync(CancellationToken cancellationToken, bool isUnitTest = false) 
    { 
        await using var scope = _serviceProvider.CreateAsyncScope(); 
        var services = scope.ResolveServices(); 

        if (isUnitTest) 
        { 
            return; 
        } 

        var queueMessages = await services.QueueService.DequeueMessagesAsync(services.AzureSettings.CmsClaimNumberRetreivalQueueName, count: 1, cancellationToken: cancellationToken); 

        if (queueMessages.Length == 0) 
        { 
            return; 
        } 

        _logger.LogInformation("Found [{Count}] CSM retrieval requests", queueMessages.Length); 

        var message = queueMessages[0]; 
        var claimRequest = DeserializeQueueMessage(message); 

        if (claimRequest == null) 
        { 
            return; 
        } 

        await ProcessClaimRetrievalRequestAsync(services, claimRequest, message, cancellationToken); 
    } 

    private async Task ProcessClaimRetrievalRequestAsync(ServiceBundle services, ClaimRetreivalQueueMessage claimRequest, QueueMessage queueMessage, CancellationToken cancellationToken) 
    { 
        _logger.LogInformation("Processing claim retrieval for CmsRef: {CmsRef}, MetaDataId: {MetaDataId}", claimRequest.CmsReferenceNumber, claimRequest.MetaDataId); 

        if (!int.TryParse(claimRequest.MetaDataId, out int id)) 
        { 
            throw new InvalidDataException($"Failed to parse metadata {claimRequest.MetaDataId}"); 
        } 

        var metaDataRecord = await services.MetaDataService.GetWorkerMetaDataByIdAsync(id, cancellationToken); 

        if (metaDataRecord.IsError) 
        { 
            _logger.LogWarning("Failed to retrieve metadata for ID: {MetaDataId}", claimRequest.MetaDataId); 
            throw new InvalidOperationException($"Failed to retrieve metadata for ID: {claimRequest.MetaDataId}"); 
        } 

        var wroiMetaData = metaDataRecord.Value.Data; 

        if (wroiMetaData is null) 
        { 
            _logger.LogWarning("Metadata record is null for ID: {MetaDataId}", claimRequest.MetaDataId); 
            throw new InvalidOperationException($"Metadata record is null for ID: {claimRequest.MetaDataId}"); 
        } 

        // Get claim and CCN data 
        var claimAndCcnResponse = await GetClaimNumberAndCcnAsync(claimRequest, services.GeneralSettings.WaitForCmsClaimCcnCallSeconds, cancellationToken); 

        if (claimAndCcnResponse != null) 
        { 
            _logger.LogInformation("Retrieved claim data for {Ref}: ClaimNumber={ClaimNumber}, CCN={CCN}", claimRequest.CmsReferenceNumber, claimAndCcnResponse.ClaimNumber, claimAndCcnResponse.CustomerCareNumber); 

            var updatedDemography = JsonConvert.DeserializeObject<Demographics>(wroiMetaData.DemographicsPayload); 

            if (updatedDemography == null) 
            { 
                _logger.LogWarning("Failed to deserialize demographics payload for MetaDataId: {MetaDataId}/ {Ref}", claimRequest.MetaDataId, claimRequest.CmsReferenceNumber); 
                throw new InvalidOperationException($"Failed to deserialize demographics for ID: {claimRequest.MetaDataId}"); 
            } 

            // 问题点：执行到这里之后，后续代码完全无动静
            await services.MetaDataService.UpdateWorkerReportOfInjuryMetaData(/* 业务参数 */);

            // 以下操作（日志、队列发布）从未执行，也无日志输出
            _logger.LogInformation("准备发布消息到下游队列");
            await services.QueueService.EnqueueMessageAsync(/* 下游队列参数 */);
        } 
    }
}

我目前想到的排查方向，也求大佬们补充： #

• 数据库更新方法的异常吞杀：打算在调用UpdateWorkerReportOfInjuryMetaData时单独加try-catch强制打日志，看看是不是这个方法内部抛出了未被捕获的异常，甚至是异步操作的异常被吞了。
• 异步死锁隐患：检查UpdateWorkerReportOfInjuryMetaData内部有没有用.Result、.Wait()这类同步阻塞代码，后台服务没有SynchronizationContext，这种操作很容易导致死锁，程序直接卡在这里。
• 作用域提前释放问题：我用了await using var scope创建作用域，会不会数据库更新后，作用域被提前Dispose，导致后续调用队列服务时触发异常但没被捕获？
• 日志输出故障：在数据库更新后加一行LogCritical级别的日志，验证是不是因为日志级别、sink配置问题导致后续日志没输出。
• 队列消息可见性超时：虽然是Dequeue消息，但如果处理时间超过队列的可见性超时，消息会重新入队，不过这应该不会导致当前线程挂掉，但还是打算检查队列的可见性配置。

内容来源于stack exchange