如何用C程序加载HTTPS网页？HTTP代码遇301跳转崩溃问题

问题概况 #

• 请求http://前缀页面正常，请求https://前缀时程序崩溃
• 去掉前缀s后收到301永久重定向，目标地址正是原HTTPS URL：

Host: a.b.c
nr 90 (90)
Full Buffer header HTTP/1.1 301 Moved Permanently
Date: Mon, 17 Oct 2022 10:02:05 GMT
Transfer-Encoding: chunked
Connection: keep-alive
Cache-Control: max-age=3600
Expires: Mon, 17 Oct 2022 11:02:05 GMT
Location: https://a.b.c/ViewDocument.aspx?fileid=12345678
Server: cloudflare

• 浏览器访问该HTTPS地址可自动下载PDF，这是程序最终目标
• 运行环境：Ubuntu 20.04
• 核心代码片段（receive函数）：

#include "stdio.h"
#include "string.h"
#include "stdlib.h"
#include <unistd.h>
#include "stdint.h"
#ifdef __linux
#include "netinet/in.h"
#include "sys/socket.h"
#include <arpa/inet.h>
#include <netdb.h>
#include <fcntl.h>
#endif
...
void receive(char *ip, char *domain, char *page){

    int sockfd, portno;
    struct sockaddr_in server_addr;
    char header[512];
    portno = 80;
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if(sockfd == -1){
        printf("error opening socket\n");
        return;
    }else{
        printf("socket opened\n");
    }

    memset((char *) &server_addr, 0, sizeof(server_addr));
    server_addr.sin_family = AF_INET;
    server_addr.sin_port = htons(portno);
#ifdef __WIN32
    server_addr.sin_addr.s_addr = inet_addr(ip);
#else
    inet_pton(AF_INET, ip, &(server_addr.sin_addr) );
#endif

    int err = 0;
    if( (err = connect(sockfd, (struct sockaddr *) &server_addr, sizeof(server_addr)) ) < 0){
        printf("failed to connect %d\n", err);
        return;
    }else{
        printf("connection successful\n");
    }

    snprintf(header, 512, "GET %s HTTP/1.1\r\nHost: %s\r\n\r\n", page, domain);
    printf("Header %s\n", header);

    int nr = send(sockfd, header, tec_string_length(header), 0);
    if(nr){
        printf("nr %d (%u)\n", nr, tec_string_length(header));
    }

    char *buf;
    buf = (char *) malloc(BUF_SIZE_INC * sizeof(char));
    memset(buf, 0, BUF_SIZE_INC);
    ssize_t bytes_received = recv(sockfd, buf, BUF_SIZE, 0);

    int size = 0;
    char *new_buffer = process_header(buf, &size);
    memmove(buf, new_buffer, BUF_SIZE_INC - (new_buffer - buf) );
    load_body(sockfd, buf, size);
    puts("*****");

#ifdef __WIN32
    closesocket(sockfd);
    WSACleanup();
#else
    close(sockfd);
#endif

}//receive*/

解决步骤 #

1. 明确崩溃原因 #

当前代码仅实现HTTP协议（端口80）的明文通信，HTTPS需要在TCP之上建立TLS/SSL加密连接（默认端口443）。直接用现有代码处理HTTPS请求，会因通信格式不匹配导致崩溃。

2. 引入OpenSSL库实现TLS加密 #

Ubuntu 20.04下通过OpenSSL库实现HTTPS通信：

• 安装OpenSSL开发包：

  sudo apt-get install libssl-dev
  

3. 修改代码支持HTTPS #

初始化/清理OpenSSL

在程序启动和退出时分别调用以下函数：

#include <openssl/ssl.h>
#include <openssl/err.h>

void init_openssl() {
    SSL_load_error_strings();
    OpenSSL_add_all_algorithms();
    SSL_library_init();
}

void cleanup_openssl() {
    EVP_cleanup();
}

改造receive函数

新增HTTPS支持逻辑，区分HTTP/HTTPS通信：

void receive(char *ip, char *domain, char *page, int is_https){
    int sockfd, portno;
    struct sockaddr_in server_addr;
    char header[512];
    portno = is_https ? 443 : 80;

    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if(sockfd == -1){
        printf("error opening socket\n");
        return;
    }

    memset((char *) &server_addr, 0, sizeof(server_addr));
    server_addr.sin_family = AF_INET;
    server_addr.sin_port = htons(portno);
#ifdef __WIN32
    server_addr.sin_addr.s_addr = inet_addr(ip);
#else
    inet_pton(AF_INET, ip, &(server_addr.sin_addr) );
#endif

    if( connect(sockfd, (struct sockaddr *) &server_addr, sizeof(server_addr)) < 0){
        printf("failed to connect\n");
        close(sockfd);
        return;
    }

    SSL_CTX *ctx = NULL;
    SSL *ssl = NULL;
    if(is_https){
        ctx = SSL_CTX_new(SSLv23_client_method());
        if(!ctx){
            printf("Failed to create SSL context\n");
            close(sockfd);
            return;
        }
        ssl = SSL_new(ctx);
        SSL_set_fd(ssl, sockfd);
        if(SSL_connect(ssl) <= 0){
            printf("TLS handshake failed\n");
            ERR_print_errors_fp(stderr);
            SSL_free(ssl);
            SSL_CTX_free(ctx);
            close(sockfd);
            return;
        }
    }

    snprintf(header, 512, "GET %s HTTP/1.1\r\nHost: %s\r\n\r\n", page, domain);
    int nr = is_https ? SSL_write(ssl, header, tec_string_length(header)) 
                      : send(sockfd, header, tec_string_length(header), 0);
    if(nr <= 0){
        printf("Failed to send data\n");
        goto cleanup;
    }

    char *buf = malloc(BUF_SIZE_INC * sizeof(char));
    memset(buf, 0, BUF_SIZE_INC);
    ssize_t bytes_received = is_https ? SSL_read(ssl, buf, BUF_SIZE) 
                                      : recv(sockfd, buf, BUF_SIZE, 0);
    if(bytes_received <= 0){
        printf("Failed to receive data\n");
        free(buf);
        goto cleanup;
    }

    int size = 0;
    char *new_buffer = process_header(buf, &size);
    memmove(buf, new_buffer, BUF_SIZE_INC - (new_buffer - buf) );
    load_body(sockfd, ssl, buf, size, is_https);
    free(buf);

cleanup:
    if(is_https){
        SSL_shutdown(ssl);
        SSL_free(ssl);
        SSL_CTX_free(ctx);
    }
#ifdef __WIN32
    closesocket(sockfd);
#else
    close(sockfd);
#endif
}

修改load_body函数适配HTTPS

在读取响应体时根据协议选择对应读取方法，并添加PDF保存逻辑：

void load_body(int sockfd, SSL *ssl, char *buf, int size, int is_https){
    ssize_t bytes_read;
    while(1){
        bytes_read = is_https ? SSL_read(ssl, buf + size, BUF_SIZE_INC - size - 1) 
                              : recv(sockfd, buf + size, BUF_SIZE_INC - size - 1, 0);
        if(bytes_read <= 0) break;
        size += bytes_read;
        buf[size] = '\0';
    }

    FILE *fp = fopen("downloaded.pdf", "wb");
    if(fp){
        fwrite(buf, 1, size, fp);
        fclose(fp);
        printf("PDF saved to downloaded.pdf\n");
    }
}

4. 处理301重定向 #

在process_header函数中解析Location字段，当检测到301响应时，自动调用改造后的receive函数发起HTTPS请求。

5. 编译程序 #

编译时链接OpenSSL库：

gcc your_program.c -o your_program -lssl -lcrypto

内容的提问来源于stack exchange，提问作者ubnewb