如何基于Polly实现WaitAndRetry失败后的30分钟重试断路器？

问题说明 #

此前误用了WaitAndRetryForeverAsync，重试模式仅应处理限流、429状态码这类瞬时故障。当订阅的API因维护下线约25分钟时，该方法以固定间隔持续重试，触发API端防火墙规则导致IP被临时封禁。

需要实现的逻辑：重试5次失败后，判定服务处于维护状态，等待30分钟后重新启用重试循环，直到重连成功（无论维护时长多久）。

原Retry类代码 #

/// <summary>
///     This class provides Transient Fault Handling extension methods.
/// </summary>
internal static class Retry
{
    public static void Do(Action action, TimeSpan retryInterval, int retryCount = 3)
    {
        _ = Do<object?>(() =>
        {
            action();
            return null;
        }, retryInterval, retryCount);
    }

    public static async Task DoAsync(Func<Task> action, TimeSpan retryInterval, int retryCount = 3)
    {
        _ = await DoAsync<object?>(async () =>
        {
            await action();
            return null;
        }, retryInterval, retryCount);
    }

    public static T Do<T>(Func<T> action, TimeSpan retryWait, int retryCount = 3)
    {
        var policyResult = Policy
            .Handle<Exception>()
            .WaitAndRetry(retryCount, retryAttempt => retryWait)
            .ExecuteAndCapture(action);

        if (policyResult.Outcome == OutcomeType.Failure)
        {
            throw policyResult.FinalException;
        }

        return policyResult.Result;
    }

    public static async Task<T> DoAsync<T>(Func<Task<T>> action, TimeSpan retryWait, int retryCount = 3)
    {
        var policyResult = await Policy
            .Handle<Exception>()
            .WaitAndRetryAsync(retryCount, retryAttempt => retryWait)
            .ExecuteAndCaptureAsync(action);

        if (policyResult.Outcome == OutcomeType.Failure)
        {
            throw policyResult.FinalException;
        }

        return policyResult.Result;
    }
}

解决方案：整合重试+断路器策略 #

要实现需求，需要将重试策略和断路器策略组合使用：

1. 先执行最多5次重试，每次间隔按需求设置；
2. 若5次重试全部失败，断路器进入打开状态，持续30分钟；
3. 30分钟后断路器进入半开状态，尝试一次调用；若失败则回到打开状态再等30分钟，若成功则关闭断路器恢复正常。

修改后的Retry类实现如下：

using Polly;
using Polly.CircuitBreaker;
using System;
using System.Threading.Tasks;

/// <summary>
///     This class provides Transient Fault Handling extension methods with circuit breaker support.
/// </summary>
internal static class RetryWithCircuitBreaker
{
    // 共享断路器策略实例，确保状态全局一致
    private static readonly AsyncCircuitBreakerPolicy _circuitBreakerPolicy = Policy
        .Handle<Exception>()
        .CircuitBreakerAsync(
            exceptionsAllowedBeforeBreaking: 1, // 半开状态下1次失败就重新打开断路器
            durationOfBreak: TimeSpan.FromMinutes(30), // 打开状态持续30分钟
            onBreak: (ex, breakDuration) =>
            {
                // 可选：记录断路器打开日志
                Console.WriteLine($"Circuit breaker opened due to exception: {ex.Message}, will retry after {breakDuration.TotalMinutes} minutes");
            },
            onReset: () =>
            {
                // 可选：记录断路器恢复日志
                Console.WriteLine("Circuit breaker reset to closed state");
            },
            onHalfOpen: () =>
            {
                // 可选：记录断路器进入半开状态日志
                Console.WriteLine("Circuit breaker entered half-open state, testing connection...");
            });

    public static void Do(Action action, TimeSpan retryInterval, int retryCount = 5)
    {
        _ = Do<object?>(() =>
        {
            action();
            return null;
        }, retryInterval, retryCount);
    }

    public static async Task DoAsync(Func<Task> action, TimeSpan retryInterval, int retryCount = 5)
    {
        _ = await DoAsync<object?>(async () =>
        {
            await action();
            return null;
        }, retryInterval, retryCount);
    }

    public static T Do<T>(Func<T> action, TimeSpan retryWait, int retryCount = 5)
    {
        // 组合重试策略和断路器策略：先重试，失败后触发断路器
        var combinedPolicy = Policy.Wrap(
            Policy.Handle<Exception>()
                .WaitAndRetry(retryCount, _ => retryWait),
            _circuitBreakerPolicy);

        try
        {
            return combinedPolicy.Execute(action);
        }
        catch (BrokenCircuitException)
        {
            // 断路器打开时会抛出此异常，递归重试直到成功
            return Do(action, retryWait, retryCount);
        }
    }

    public static async Task<T> DoAsync<T>(Func<Task<T>> action, TimeSpan retryWait, int retryCount = 5)
    {
        // 组合异步重试策略和异步断路器策略
        var combinedPolicy = Policy.WrapAsync(
            Policy.Handle<Exception>()
                .WaitAndRetryAsync(retryCount, _ => retryWait),
            _circuitBreakerPolicy);

        try
        {
            return await combinedPolicy.ExecuteAsync(action);
        }
        catch (BrokenCircuitException)
        {
            // 断路器打开时，等待30分钟后再重试，避免无效递归
            await Task.Delay(TimeSpan.FromMinutes(30));
            return await DoAsync(action, retryWait, retryCount);
        }
    }
}

关键说明 #

• 共享断路器实例：确保整个应用中断路器状态一致，避免多个实例导致状态混乱；
• 策略组合：使用Wrap/WrapAsync将重试策略和断路器策略组合，执行顺序是先重试，重试失败后触发断路器；
• 断路器逻辑：设置exceptionsAllowedBeforeBreaking:1，意味着半开状态下只要一次调用失败，就重新打开断路器；durationOfBreak:30分钟控制维护状态的等待时长；
• 循环重试：捕获BrokenCircuitException后，异步版本添加Task.Delay等待30分钟再重试，同步版本直接递归，确保持续尝试直到服务恢复。

内容的提问来源于stack exchange，提问作者nop