Python gRPC服务端与客户端连接失败，请求排查故障

问题场景 #

搭建gRPC服务端与客户端时，客户端调用始终报错无法连接，相关代码如下：

服务端代码

import logging
import grpc
import sys
sys.path.append("proto")
import proto.nvidia_pb2_grpc
from servicer import NvidiaServicer


GRPC_PORT = '50057'
socket = "localhost:{0}".format(GRPC_PORT)

def server():
    logger.info('Setting up gRPC server')
    grpc_server = grpc.server(concurrent.futures.ThreadPoolExecutor(max_workers=10))
    proto.nvidia_pb2_grpc.add_NvidiaServicer_to_server(
        NvidiaServicer(), grpc_server
    )
    
    logger.info(f'Starting server at {socket}')
    grpc_server.add_insecure_port(socket)
    grpc_server.start()

服务实现类代码

import proto.nvidia_pb2
import proto.nvidia_pb2_grpc
import logging
from driver_status import checkDriverStatus
logger = logging.getLogger()

class NvidiaServicer(proto.nvidia_pb2_grpc.NvidiaServicer):
    def NvidiaDriverStatus(self, request, context):
        logger.info('######################################################')
        logger.info('gRPC server got request to check driver status')
        response = proto.nvidia_pb2.DriverStatus()
        result = checkDriverStatus()

        response.status.value = result
        return response

客户端代码

from asyncio.log import logger
import grpc
import os
from proto import nvidia_pb2
from proto import nvidia_pb2_grpc

GRPC_PORT = '50057'
socket = 'localhost:{}'.format(GRPC_PORT)

def GrpcClientNvidia():
    try:
        if os.environ.get('https_proxy'):
            del os.environ['https_proxy']
        if os.environ.get('http_proxy'):
            del os.environ['http_proxy']
        channel = grpc.insecure_channel(socket, options=(('grpc.enable_http_proxy', 0),))
        stub = nvidia_pb2_grpc.NvidiaStub(channel)
        res = stub.NvidiaDriverStatus(nvidia_pb2.Empty())
        return res.status.value
    except grpc.FutureTimeoutError:
         logger.error('Error connecting to nvidia server')

主程序代码

from GrpcClientNvidia import GrpcClientNvidia
result = {}
result['SW'] = {}
result['SW']['nvidia'] = runNvidiaDriverCheck()

def runNvidiaDriverCheck():
    return GrpcClientNvidia()

报错信息 #

File "/usr/local/lib/python3.6/site-packages/grpc/_channel.py", line 849, in _end_unary_response_blocking
    raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
        status = StatusCode.UNAVAILABLE
        details = "failed to connect to all addresses"
        debug_error_string = "{\"created\":\"@1661349318.910632939\",\"description\":\"Failed to pick subchannel\",\"file\":\"src/core/ext/filters/client_channel/client_channel.cc\",\"file_line\":3260,\"referenced_errors\":[{\"created\":\"@1661349318.910631609\",\"description\":\"failed to connect to all addresses\",\"file\":\"src/core/lib/transport/error_utils.cc\",\"file_line\":167,\"grpc_status\":14}]}"

问题排查与修复 #

1. 服务端未持续运行
   gRPC的grpc_server.start()方法是非阻塞的，调用后主线程会直接退出，导致服务启动后立刻终止。同时原代码缺少concurrent.futures的导入，修复如下：

   import logging
   import grpc
   import sys
   import concurrent.futures
   import time
   sys.path.append("proto")
   import proto.nvidia_pb2_grpc
   from servicer import NvidiaServicer
   
   # 初始化日志
   logging.basicConfig(level=logging.INFO)
   logger = logging.getLogger(__name__)
   
   GRPC_PORT = '50057'
   socket = "localhost:{0}".format(GRPC_PORT)
   
   def server():
       logger.info('Setting up gRPC server')
       grpc_server = grpc.server(concurrent.futures.ThreadPoolExecutor(max_workers=10))
       proto.nvidia_pb2_grpc.add_NvidiaServicer_to_server(
           NvidiaServicer(), grpc_server
       )
       
       logger.info(f'Starting server at {socket}')
       grpc_server.add_insecure_port(socket)
       grpc_server.start()
       
       # 阻塞主线程，保持服务运行
       try:
           while True:
               time.sleep(86400)
       except KeyboardInterrupt:
           grpc_server.stop(0)
   

2. 服务端日志未初始化
   原服务端未初始化日志系统，无法确认服务是否正常启动，添加logging.basicConfig(level=logging.INFO)和logger = logging.getLogger(__name__)即可解决。

3. 主程序函数调用顺序错误
   Python要求函数先定义再调用，调整主程序代码顺序：

   from GrpcClientNvidia import GrpcClientNvidia
   
   def runNvidiaDriverCheck():
       return GrpcClientNvidia()
   
   result = {}
   result['SW'] = {}
   result['SW']['nvidia'] = runNvidiaDriverCheck()
   

4. 客户端异常捕获不全
   原客户端只捕获了超时错误，无法覆盖连接类异常，补充捕获grpc.RpcError：

   def GrpcClientNvidia():
       try:
           if os.environ.get('https_proxy'):
               del os.environ['https_proxy']
           if os.environ.get('http_proxy'):
               del os.environ['http_proxy']
           channel = grpc.insecure_channel(socket, options=(('grpc.enable_http_proxy', 0),))
           stub = nvidia_pb2_grpc.NvidiaStub(channel)
           res = stub.NvidiaDriverStatus(nvidia_pb2.Empty())
           return res.status.value
       except grpc.FutureTimeoutError:
            logger.error('Error connecting to nvidia server: Timeout')
       except grpc.RpcError as e:
            logger.error(f'gRPC connection error: {e.code()} - {e.details()}')
   

验证步骤 #

1. 启动修改后的gRPC服务端，确认日志打印"Starting server at localhost:50057"
2. 运行主程序，检查客户端是否能正常连接并获取结果

内容的提问来源于stack exchange，提问作者Yuki1112