生产环境部分Pod随机触发'int' object is not callable异常求助

生产环境中部分Pod会随机触发Python库异常，并非所有Pod都会出现。所有DynamoDB调用都会触发该异常，代码正常运行一个月后突然出现此问题。已添加追踪打印，考虑升级urllib3版本，但不确定能否解决，恳请协助排查。

错误追踪信息 #

Traceback (most recent call last):
  File "/usr/local/lib/python3.10/site-packages/botocore/httpsession.py", line 448, in send
    urllib_response = conn.urlopen(
  File "/usr/local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 703, in urlopen
    httplib_response = self._make_request(
  File "/usr/local/lib/python3.10/site-packages/newrelic/hooks/external_urllib3.py", line 32, in _nr_wrapper_make_request_
    return wrapped(*args, **kwargs)
  File "/usr/local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 386, in _make_request
    self._validate_conn(conn)
  File "/usr/local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 1040, in _validate_conn
    conn.connect()
  File "/usr/local/lib/python3.10/site-packages/urllib3/connection.py", line 416, in connect
    self.sock = ssl_wrap_socket(
  File "/usr/local/lib/python3.10/site-packages/urllib3/util/ssl_.py", line 424, in ssl_wrap_socket
    context.set_alpn_protocols(ALPN_PROTOCOLS)
  File "/usr/local/lib/python3.10/ssl.py", line 566, in set_alpn_protocols
    if len(b) == 0 or len(b) > 255:
TypeError: 'int' object is not callable

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/code/app/aws_utils/dynamo_db.py", line 176, in update_multi_attributes
    response = table.update_item(
  File "/usr/local/lib/python3.10/site-packages/boto3/resources/factory.py", line 580, in do_action
    response = action(self, *args, **kwargs)
  File "/usr/local/lib/python3.10/site-packages/boto3/resources/action.py", line 88, in __call__
    response = getattr(parent.meta.client, operation_name)(*args, **params)
  File "/usr/local/lib/python3.10/site-packages/botocore/client.py", line 514, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/usr/local/lib/python3.10/site-packages/botocore/client.py", line 921, in _make_api_call
    http, parsed_response = self._make_request(
  File "/usr/local/lib/python3.10/site-packages/botocore/client.py", line 944, in _make_request
    return self._endpoint.make_request(operation_model, request_dict)
  File "/usr/local/lib/python3.10/site-packages/newrelic/hooks/external_botocore.py", line 108, in _nr_endpoint_make_request_
    result = wrapped(*args, **kwargs)
  File "/usr/local/lib/python3.10/site-packages/botocore/endpoint.py", line 119, in make_request
    return self._send_request(request_dict, operation_model)
  File "/usr/local/lib/python3.10/site-packages/botocore/endpoint.py", line 231, in _send_request
    raise exception
  File "/usr/local/lib/python3.10/site-packages/botocore/endpoint.py", line 281, in _do_get_response
    http_response = self._send(request)
  File "/usr/local/lib/python3.10/site-packages/botocore/endpoint.py", line 377, in _send
    return self.http_session.send(request)
  File "/usr/local/lib/python3.10/site-packages/botocore/httpsession.py", line 493, in send
    raise HTTPClientError(error=e)
botocore.exceptions.HTTPClientError: An HTTP Client raised an unhandled exception: 'int' object is not callable

相关代码片段 #

def update_multi_attributes(
        self,
        table,
        partition_key,
        sort_key,
        attr_value_dict,
        delete_attr=[],
    ):
        try:
            if not partition_key or not sort_key:
                return None
            if not attr_value_dict and len(delete_attr) == 0:
                return None
            if not attr_value_dict and len(delete_attr) > 0:
                return self.delete_attributes(
                    table=table,
                    partition_key=partition_key,
                    sort_key=sort_key,
                    delete_attr=delete_attr,
                )
            expression_attribute_values = {}
            expression_attribute_names = {}
            update_expression = "SET "
            remove_attr_expression = "REMOVE "
            index = 0
            attr_value_dict.pop("partition_key", None)
            attr_value_dict.pop("sort_key", None)

            for key, value in attr_value_dict.items():
                index += 1
                var_name = "#var" + str(index)
                var_value_name = ":var" + str(index)
                expression_attribute_values[var_value_name] = value
                expression_attribute_names[var_name] = key
                if index < len(attr_value_dict.keys()):
                    update_expression = (
                        update_expression + var_name + "=" + var_value_name + ","
                    )
                else:
                    update_expression = (
                        update_expression + var_name + "=" + var_value_name
                    )
            index = 0
            for attr in delete_attr:
                index += 1
                var_name = "#rvar" + str(index)
                expression_attribute_names[var_name] = attr
                if index < len(delete_attr):
                    remove_attr_expression = remove_attr_expression + var_name + ","
                else:
                    remove_attr_expression = remove_attr_expression + var_name
            if len(delete_attr) > 0:
                update_expression = update_expression + " " + remove_attr_expression
            response = table.update_item(
                Key=self.get_key(partition_key, sort_key),
                ExpressionAttributeNames=expression_attribute_names,
                UpdateExpression=update_expression,
                ConditionExpression="attribute_exists(partition_key)"
                "and attribute_exists(sort_key)",
                ExpressionAttributeValues=expression_attribute_values,
            )
            return response
        except botocore.exceptions.ClientError as e:
            logger.info(
                "Dynamo: Error Update MultiAttribute boto3{} error{},{}for{},{}".format(
                    attr_value_dict,
                    e,
                    e.response["Error"],
                    partition_key,
                    traceback.format_exc(),
                )
            )
        except Exception as e:
            logger.info(
                "Dynamo: Error Update MultiAttribute {} error {} for {}, {}".format(
                    attr_value_dict, e, partition_key, traceback.format_exc()
                )
            )
        return None

使用的包版本 #

• urllib3-1.26.8
• botocore-1.27.77
• boto3-1.24.77

内容的提问来源于stack exchange，提问作者Manish Jain