C#异步任务中退避与取消重试逻辑实现问题排查

问题场景 #

我维护一个后台运行的服务，负责调用外部服务执行客户档案更新，需求是：

• 对目标服务的HTTP请求先做3次重试
• 3次全失败后，退避指定分钟数再执行第4次尝试
• 第4次仍失败则终止当前同步流程，不再处理后续客户档案

但实际运行时，即使3次重试全部失败，退避后的第4次请求日志始终不输出，且10分钟退避完全没执行，反而直接跳到下一个客户档案，重新进入AttemptUpdate方法。

现有代码 #

主同步方法 #

private async Task DoSyncCustomerProfile(SyncMessage message)
{
    foreach (var customerProfile in message.CustomerProfilesDto)
    {
        var cts = new CancellationTokenSource();
        var cancellationToken = cts.Token;

        // 创建同步数据的任务委托
        Func<Task<RequestResponse<SomeResponse>>> task = async () =>
        {
            var url = $"https://google.com/v1/abcdefhj";
            var response = await _requestService.Post<SomeResponse>(
                url,
                customerProfile,
                nameof(SomeOperation),
                _auth,
                "Bearer"
            );
            return new RequestResponse<SomeResponse>
            {
                Data = response.Data,
                Successful = response.Successful,
                Message = response.Message
            };
        };

        var response = await AttemptUpdate(task, cancellationToken, _options.Value.MaxRetries);

        if (!response.Successful)
        {
            _logger.LogDebug("设置退避时间：{BackoffTime} 分钟", _options.Value.BackoffTime);

            // 执行退避
            await Task.Delay(TimeSpan.FromMinutes(_options.Value.BackoffTime), cancellationToken);

            var url = $"https://google.com/v1/abcdefghij";
            var responseFromRetry = await _requestService.Post<SomeResponse>(
                url,
                customerProfile,
                nameof(SomeAction),
                _auth,
                "Bearer"
            );
            _logger.LogDebug("退避后重试响应：{Response}", JsonConvert.SerializeObject(responseFromRetry));
            response = new RequestResponse<SomeResponse>
            {
                Data = responseFromRetry.Data,
                Successful = responseFromRetry.Successful,
                Message = responseFromRetry.Message
            };

            if (!response.Successful)
            {
                _logger.LogDebug("3次尝试后仍同步失败，跳过后续档案");
                break;
            }
        }
    }
}

重试方法 #

private async Task<RequestResponse<SomeResponse>> AttemptUpdate(
        Func<Task<RequestResponse<SomeResponse>>> task,
        CancellationToken cancellationToken,
        int maxAttemptCount = 3)
{
    for (var attempted = 0; attempted < maxAttemptCount; attempted++)
    {
        _logger.LogDebug("第 {Attempted} 次尝试同步客户档案", attempted + 1);

        // 检查是否取消
        if (cancellationToken.IsCancellationRequested)
        {
            _logger.LogDebug("收到取消请求，退出重试循环");
            return new RequestResponse<SomeResponse>
            {
                Successful = false,
                Message = "操作已取消"
            };
        }
        var result = await task();
        _logger.LogDebug("收到响应：{Response}", JsonConvert.SerializeObject(result));

        // 成功则直接返回
        if (result.Successful)
        {
            _logger.LogDebug("客户档案同步成功，响应：{Response}", result.Data?.Message);
            return result;
        }

        _logger.LogDebug("客户档案同步失败，响应：{Response}", result.Data?.Message);
    }

    return new RequestResponse<SomeResponse>
    {
        Successful = false,
        Message = "客户档案同步失败"
    };
}

问题根源 #

1. 取消令牌被意外触发：每次循环创建的CancellationTokenSource没有绑定到明确的取消逻辑，若后台作业框架或隐性逻辑触发取消，会导致AttemptUpdate提前返回，且后续Task.Delay会因取消令牌触发直接跳过，完全不执行退避和第4次请求。
2. 日志级别配置问题：LogDebug级别的日志默认可能未开启，即使代码执行了，日志也不会输出到控制台或文件。
3. 逻辑描述错误：当前代码中break注释写的是"3次尝试失败"，但实际是第4次失败才会触发，容易混淆逻辑判断。

修复方案 #

方案1：移除不必要的取消令牌 #

如果没有明确的取消需求，直接移除CancellationTokenSource，避免意外取消打断流程：

private async Task DoSyncCustomerProfile(SyncMessage message)
{
    foreach (var customerProfile in message.CustomerProfilesDto)
    {
        // 创建同步数据的任务委托
        Func<Task<RequestResponse<SomeResponse>>> task = async () =>
        {
            var url = $"https://google.com/v1/abcdefhj";
            var response = await _requestService.Post<SomeResponse>(
                url,
                customerProfile,
                nameof(SomeOperation),
                _auth,
                "Bearer"
            );
            return new RequestResponse<SomeResponse>
            {
                Data = response.Data,
                Successful = response.Successful,
                Message = response.Message
            };
        };

        var response = await AttemptUpdate(task, CancellationToken.None, _options.Value.MaxRetries);

        if (!response.Successful)
        {
            _logger.LogDebug("设置退避时间：{BackoffTime} 分钟", _options.Value.BackoffTime);

            // 执行退避（无取消令牌）
            await Task.Delay(TimeSpan.FromMinutes(_options.Value.BackoffTime));

            var url = $"https://google.com/v1/abcdefghij";
            var responseFromRetry = await _requestService.Post<SomeResponse>(
                url,
                customerProfile,
                nameof(SomeAction),
                _auth,
                "Bearer"
            );
            _logger.LogDebug("退避后重试响应：{Response}", JsonConvert.SerializeObject(responseFromRetry));
            response = new RequestResponse<SomeResponse>
            {
                Data = responseFromRetry.Data,
                Successful = responseFromRetry.Successful,
                Message = responseFromRetry.Message
            };

            if (!response.Successful)
            {
                _logger.LogDebug("4次尝试后仍同步失败，终止后续档案处理");
                break;
            }
        }
    }
}

方案2：开启Debug级别日志 #

修改appsettings.json中的日志配置，确保Debug级别日志被输出：

{
  "Logging": {
    "LogLevel": {
      "Default": "Debug",
      "Microsoft": "Warning",
      "Microsoft.Hosting.Lifetime": "Information"
    }
  }
}

方案3：统一重试逻辑（推荐） #

把前3次重试+退避+第4次尝试的逻辑整合到一个方法中，避免分散逻辑导致的错误：

private async Task<RequestResponse<SomeResponse>> AttemptUpdateWithBackoff(
        Func<Task<RequestResponse<SomeResponse>>> normalTask,
        Func<Task<RequestResponse<SomeResponse>>> backoffTask,
        CancellationToken cancellationToken,
        int maxNormalRetries = 3,
        int backoffMinutes = 10)
{
    // 前3次常规重试
    for (var attempt = 0; attempt < maxNormalRetries; attempt++)
    {
        _logger.LogDebug("第 {Attempt} 次常规尝试同步客户档案", attempt + 1);

        if (cancellationToken.IsCancellationRequested)
        {
            _logger.LogDebug("收到取消请求，终止重试");
            return new RequestResponse<SomeResponse> { Successful = false, Message = "操作已取消" };
        }

        var result = await normalTask();
        _logger.LogDebug("常规尝试响应：{Response}", JsonConvert.SerializeObject(result));

        if (result.Successful)
        {
            _logger.LogDebug("常规尝试成功");
            return result;
        }
    }

    // 3次失败后执行退避
    _logger.LogDebug("3次常规尝试失败，开始退避 {BackoffMinutes} 分钟", backoffMinutes);
    await Task.Delay(TimeSpan.FromMinutes(backoffMinutes), cancellationToken);

    // 第4次退避后尝试
    _logger.LogDebug("执行退避后的第4次尝试");
    var backoffResult = await backoffTask();
    _logger.LogDebug("退避后尝试响应：{Response}", JsonConvert.SerializeObject(backoffResult));

    if (backoffResult.Successful)
    {
        _logger.LogDebug("退避后尝试成功");
        return backoffResult;
    }

    _logger.LogDebug("4次尝试全部失败");
    return new RequestResponse<SomeResponse> { Successful = false, Message = "4次尝试均失败" };
}

主方法调用简化为：

private async Task DoSyncCustomerProfile(SyncMessage message)
{
    foreach (var customerProfile in message.CustomerProfilesDto)
    {
        // 常规请求任务
        var normalTask = async () =>
        {
            var url = $"https://google.com/v1/abcdefhj";
            var response = await _requestService.Post<SomeResponse>(
                url, customerProfile, nameof(SomeOperation), _auth, "Bearer");
            return new RequestResponse<SomeResponse>
            {
                Data = response.Data, Successful = response.Successful, Message = response.Message
            };
        };

        // 退避后的请求任务
        var backoffTask = async () =>
        {
            var url = $"https://google.com/v1/abcdefghij";
            var response = await _requestService.Post<SomeResponse>(
                url, customerProfile, nameof(SomeAction), _auth, "Bearer");
            return new RequestResponse<SomeResponse>
            {
                Data = response.Data, Successful = response.Successful, Message = response.Message
            };
        };

        var finalResponse = await AttemptUpdateWithBackoff(
            normalTask, backoffTask, CancellationToken.None,
            _options.Value.MaxRetries, _options.Value.BackoffTime);

        if (!finalResponse.Successful)
        {
            _logger.LogDebug("终止后续客户档案同步");
            break;
        }
    }
}

验证步骤 #

1. 构造外部服务持续失败的测试场景，确认退避时间是否正常执行
2. 检查日志输出，确认所有Debug级别的日志都能正常显示
3. 验证第4次失败后是否会终止后续档案处理

内容的提问来源于stack exchange，提问作者Kwaku Biney