Envoy Proxy持续返回CANCELLED错误，配置无限超时仍未解决

这个问题已持续数月，目前只能靠过度扩容服务器（30最大并发请求对应60个实例）缓解。即便将客户端及Envoy配置中所有超时设为无限大，客户端仍会收到CANCELLED请求，自定义客户端也存在同样问题。已配置重试策略、超大超时值等，但均无效。/stats及Prometheus指标未显示错误原因，仅偶见重试记录。请求为异步调用，耗时1-10分钟，后端服务Pod无异常，考虑切换至Linkerd。

admin:
  address:
    socket_address:
      protocol: TCP
      address: 0.0.0.0
      port_value: 9901

static_resources:
  listeners:
  - name: listener_0
    address:
      socket_address:
        protocol: TCP
        address: 0.0.0.0
        port_value: 50051
    listener_filters:
    - name: "envoy.filters.listener.tls_inspector"
      typed_config:
        "@type": type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector
    filter_chains:
    - filters:
      - name: envoy.filters.network.rbac
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC
          rules:
            policies:
              "check-client-cn":
                permissions:
                  - any: true
                principals:
                  - authenticated:
                      principal_name:
                        exact: "XX-XX-XX"
                  - authenticated:
                      principal_name:
                        exact: "CN=XX-XX"
                  - authenticated:
                      principal_name:
                        exact: "CN=client"
          stat_prefix: rbac
      - name: envoy.filters.network.http_connection_manager
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
          access_log:
          - name: envoy.access_loggers.stdout
            typed_config:
              "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
          codec_type: AUTO
          stat_prefix: ingress_http
          route_config:
            name: local_route
            virtual_hosts:
            - name: local_service
              domains:
              - "*"
              routes:
              - match:
                  prefix: "/"
                route:
                  cluster: service-grpc-tls
                  timeout: 2700s
                  retry_policy:
                    retry_on: cancelled,deadline-exceeded,resource-exhausted
                    num_retries: 3
                    per_try_timeout: 900s
          http_filters:
          - name: envoy.filters.http.router
            typed_config:
              "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
      transport_socket:
        name: envoy.transport_sockets.tls
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext
          require_client_certificate: true
          common_tls_context:
            alpn_protocols: h2
            validation_context:
              trusted_ca:
                filename: /etc/ssl/envoy/ca.crt
              allow_expired_certificate: true
            tls_certificates:
            - certificate_chain:
                filename: /etc/ssl/envoy/server.crt
              private_key:
                filename: /etc/ssl/envoy/server.key

  clusters:
  - name: service-grpc-tls
    connect_timeout: 10s
    type: STRICT_DNS
    dns_lookup_family: V4_ONLY
    lb_policy: ROUND_ROBIN
    http2_protocol_options:
      max_concurrent_streams: 10000
    upstream_connection_options:
      tcp_keepalive: {}
    circuit_breakers:
      thresholds:
      - priority: DEFAULT
        max_requests: 10000
        max_pending_requests: 10000
        max_retries: 3
    load_assignment:
      cluster_name: service-grpc-tls
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: XXX-XXX.XXX-XXX
                port_value: 8080

内容的提问来源于stack exchange，提问作者amro4934