基于OpenSSL的客户端认证TLS握手失败问题求助

正在开发基于OpenSSL的客户端与服务端程序，预期服务端完成客户端认证后，双方实现问候消息交互，服务端监听12345端口。但运行时服务端SSL握手失败，客户端显示握手成功却收不到有效响应。

服务端代码 #

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <dirent.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <openssl/x509.h>
#include <openssl/rand.h>
#include <netinet/in.h>
#include <sys/socket.h>

#define PORT 12345

void initialize_openssl() {
        OpenSSL_add_all_algorithms();
        SSL_load_error_strings();
        OpenSSL_add_ssl_algorithms();
}

void cleanup_openssl() {
        EVP_cleanup();
}

SSL_CTX* create_server_context() {
        const SSL_METHOD *method = TLS_server_method();
        SSL_CTX *ctx = SSL_CTX_new(method);

        if (!ctx) {
                perror("Unable to create SSL context");
                ERR_print_errors_fp(stderr);
                exit(EXIT_FAILURE);
        }

        return ctx;
}

void configure_server_context(SSL_CTX *ctx) {
        /* Load server certificate and private key */
        if (SSL_CTX_use_certificate_file(ctx, "/certs/server.crt", SSL_FILETYPE_PEM) <= 0) {
                perror("Unable to load server certificate");
                ERR_print_errors_fp(stderr);
                exit(EXIT_FAILURE);
        }

        if (SSL_CTX_use_PrivateKey_file(ctx, "/certs/server.key", SSL_FILETYPE_PEM) <= 0) {
                perror("Unable to load server private key");
                ERR_print_errors_fp(stderr);
                exit(EXIT_FAILURE);
        }

        /* Verify private key matches the certificate */
        if (!SSL_CTX_check_private_key(ctx)) {
                fprintf(stderr, "Private key does not match the certificate public key\n");
                exit(EXIT_FAILURE);
        }

        /* Load the directory containing client certificates */
        if (SSL_CTX_load_verify_locations(ctx, NULL, "/certs/") <= 0) {
                perror("SSL_CTX_load_verify_locations failed for /certs directory");
                exit(1);
        }

        /* Set the verification mode to allow client certificates */
        /*SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL);*/
        SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, NULL);
        SSL_CTX_set_verify_depth(ctx, 4);
}
    
void handle_client(SSL *ssl) {
        char buf[1024];
        int bytes;

        /* Receive data from the client */
        bytes = SSL_read(ssl, buf, sizeof(buf));
        if (bytes < 0) {
                perror("SSL_read failed");
                ERR_print_errors_fp(stderr);
        } else {
                buf[bytes] = 0;
                printf("Received message: %s\n", buf);
        }

        /* Send a response to the client */
        const char *response = "Hello from server!";
        SSL_write(ssl, response, strlen(response));
}

int main() {
        int sockfd, client_sock;
        struct sockaddr_in addr;
        SSL_CTX *ctx;
        SSL *ssl;

        initialize_openssl();
        ctx = create_server_context();
        configure_server_context(ctx);

        /* Create server socket */
        sockfd = socket(AF_INET, SOCK_STREAM, 0);
        if (sockfd == -1) {
                perror("Unable to create socket");
                exit(EXIT_FAILURE);
        }

        addr.sin_family = AF_INET;
        addr.sin_addr.s_addr = INADDR_ANY;
        addr.sin_port = htons(PORT);

        if (bind(sockfd, (struct sockaddr*)&addr, sizeof(addr)) == -1) {
                perror("Unable to bind");
                exit(EXIT_FAILURE);
        }

        if (listen(sockfd, 1) == -1) {
                perror("Unable to listen");
                exit(EXIT_FAILURE);
        }

        printf("Waiting for connections on port %d...\n", PORT);

        /* Accept client connection */
        client_sock = accept(sockfd, NULL, NULL);
        if (client_sock == -1) {
                perror("Unable to accept connection");
                exit(EXIT_FAILURE);
        }

        /* Create SSL object and associate it with the client socket */
        ssl = SSL_new(ctx);
        SSL_set_fd(ssl, client_sock);

        if (SSL_accept(ssl) == -1) {
                perror("SSL_accept failed");
                ERR_print_errors_fp(stderr);
        } else {
                printf("SSL handshake successful\n");

                /* Extract the client certificate */
                X509 *client_cert = SSL_get_peer_certificate(ssl);
                if (client_cert == NULL) {
                        fprintf(stderr, "Client did not provide a certificate\n");
                        SSL_shutdown(ssl);
                        SSL_free(ssl);
                        close(client_sock);
                        return 0;
                }

                handle_client(ssl);

                X509_free(client_cert); /* Free the client certificate after use */
        }

        /* Close the server socket */
        close(client_sock);
        close(sockfd);
        SSL_CTX_free(ctx);
        cleanup_openssl();
        return 0;
}

客户端代码 #

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <dirent.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <openssl/x509.h>
#include <openssl/rand.h>
#include <netinet/in.h>
#include <sys/socket.h>

#define SERVER "10.32.151.173"
#define PORT 12345

void initialize_openssl() {
        OpenSSL_add_all_algorithms();
        SSL_load_error_strings();
        OpenSSL_add_ssl_algorithms();
}

void cleanup_openssl() {
        EVP_cleanup();
}

SSL_CTX* create_client_context() {
        const SSL_METHOD *method = TLS_client_method();
        SSL_CTX *ctx = SSL_CTX_new(method);

        if (!ctx) {
                perror("Unable to create SSL context");
                ERR_print_errors_fp(stderr);
                exit(EXIT_FAILURE);
        }

        return ctx;
}

void configure_client_context(SSL_CTX *ctx) {
        /* Load the client's certificate and private key */
        if (SSL_CTX_use_certificate_file(ctx, "/certs/client.crt", SSL_FILETYPE_PEM) <= 0) {
                perror("Unable to load client certificate");
                ERR_print_errors_fp(stderr);
                exit(EXIT_FAILURE);
        }

        if (SSL_CTX_use_PrivateKey_file(ctx, "/certs/client.key", SSL_FILETYPE_PEM) <= 0) {
                perror("Unable to load client private key");
                ERR_print_errors_fp(stderr);
                exit(EXIT_FAILURE);
        }

        /* Verify the client's private key matches the certificate */
        if (!SSL_CTX_check_private_key(ctx)) {
                fprintf(stderr, "Private key does not match the certificate public key\n");
                exit(EXIT_FAILURE);
        }

        /* Load the server's certificate to verify the server's identity */
        if (SSL_CTX_load_verify_locations(ctx, NULL, "/certs/") <= 0) {
                perror("SSL_CTX_load_verify_locations failed for client certs directory");
                exit(1);
        }

        /* Set the verification mode to verify server certificates */
        SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL);
        SSL_CTX_set_verify_depth(ctx, 4);
}

void communicate_with_server(SSL *ssl) {
        char *msg = "Hello from client!";
        SSL_write(ssl, msg, strlen(msg));

        char buf[1024];
        int bytes = SSL_read(ssl, buf, sizeof(buf));
        if (bytes < 0) {
                perror("SSL_read failed");
                ERR_print_errors_fp(stderr);
        } else {
                buf[bytes] = 0;
                printf("Received from server: %s\n", buf);
        }
}

int main() {
        int sockfd;
        struct sockaddr_in addr;
        SSL_CTX *ctx;
        SSL *ssl;

        initialize_openssl();
        ctx = create_client_context();
        configure_client_context(ctx);

        /* Create socket and connect to the server */
        sockfd = socket(AF_INET, SOCK_STREAM, 0);
        if (sockfd == -1) {
                perror("Unable to create socket");
                exit(EXIT_FAILURE);
        }

        addr.sin_family = AF_INET;
        addr.sin_port = htons(PORT);
        addr.sin_addr.s_addr = inet_addr(SERVER);

        if (connect(sockfd, (struct sockaddr*)&addr, sizeof(addr)) == -1) {
                perror("Unable to connect");
                exit(EXIT_FAILURE);
        }

        /* Create SSL object and associate it with the socket */
        ssl = SSL_new(ctx);
        SSL_set_fd(ssl, sockfd);

        if (SSL_connect(ssl) == -1) {
                perror("SSL_connect failed");
                ERR_print_errors_fp(stderr);
        } else {
                printf("SSL handshake successful\n");

                /* Perform communication with the server */
                communicate_with_server(ssl);
        }

        /* Close the SSL connection and the socket */
        SSL_shutdown(ssl);
        SSL_free(ssl);
        close(sockfd);
        SSL_CTX_free(ctx);
        cleanup_openssl();
        return 0;
}

证书生成步骤 #

# Generate the CA certificate
openssl genpkey -algorithm RSA -out ca.key
openssl req -new -x509 -key ca.key -out ca.crt -days 365

# Generate the server certificate and key
openssl genpkey -algorithm RSA -out server.key
openssl req -new -key server.key -out server.csr
openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 365

# Generate the client certificate and key
openssl genpkey -algorithm RSA -out client.key
openssl req -new -key client.key -out client.csr
openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out client.crt -days 365

证书文件分布 #

• 服务端/certs目录：ca.crt、ca.key、ca.srl、client.crt、client.csr、client.key、server.crt、server.csr、server.key
• 客户端/certs目录：client.crt、client.csr、client.key

运行报错信息 #

服务端输出 #

Waiting for connections on port 12345...
SSL_accept failed: Error 0
00000001:error:0A000086:SSL routines:(unknown function):certificate verify failed:ssl/statem/statem_srvr.c:3523:

客户端输出 #

SSL handshake successful
Received from server:

排查思路与解决方案 #

1. 服务端证书目录未生成哈希链接 #

OpenSSL通过目录加载信任证书时，需要目录内存在证书哈希值的符号链接（由c_rehash生成），否则无法找到CA证书验证客户端证书链。

• 修复：在服务端/certs目录执行命令：

  cd /certs
  c_rehash .
  

2. 客户端缺少CA证书 #

客户端当前/certs目录无ca.crt，无法验证服务端证书合法性，导致握手逻辑异常。

• 修复：将ca.crt复制到客户端/certs目录。

3. 服务端客户端认证模式未强制 #

当前服务端设置SSL_VERIFY_PEER，仅请求客户端证书但不强制验证通过，若需要严格认证，应启用强制模式：

• 修改服务端代码：

  SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL);
  

4. 验证证书链有效性 #

确认客户端/服务端证书能被CA证书验证：

• 在服务端执行：

  openssl verify -CAfile /certs/ca.crt /certs/client.crt
  openssl verify -CAfile /certs/ca.crt /certs/server.crt
  

  若返回OK则证书链有效，否则重新生成证书。

5. 完善代码错误处理 #

• 服务端SSL_accept失败后，立即关闭客户端套接字，避免客户端继续发送无效数据；
• 客户端SSL_connect返回-1时，终止后续通信逻辑。

内容的提问来源于stack exchange，提问作者babon