C++ Socket HTTP客户端响应体数据异常问题求助

问题描述 #

开发基于HTTP/1.1的C++ Socket客户端（仅HTTP 80端口），用于下载网站数据。程序能建立连接并获取响应，但写入文件的内容与wget/浏览器结果不一致：存在多余空格，且缺失末尾</html>标签。已通过strstr去除响应头，但问题仍存在。

测试详情 #

• 测试网址：http://www.testingmcafeesites.com/testcat_ac.html
• wget命令：wget http://www.testingmcafeesites.com/testcat_ac.html，获取内容完整包含</html>标签
• 程序生成的文件存在多余内容，缺失末尾标签

原代码 #

#include <fstream>
#include <iostream>
#include <sys/socket.h>
#include <sys/types.h>
#include <netdb.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <string>
#include <cstring>

using namespace std;

string resolveToIPAddress(string, string);
int getPortNum(string);
string getUrlPath(string);
string getDomain(string);
string getDomainWithFile(string);
string getUrlWithNoPort(string);
string getFileFromURL(string);
string getFilePath(string);
bool hasHTTPProtocol(string);
bool hasPortNum(string);
bool checkIfValidProtocol(string);


int main(int argc, char ** argv) {
    //Create a file to be opened for output
    ofstream out_file;
    out_file.open("output", ios::out | ios::binary);
    int connection_port = 0, send_req, recv_req;
    string ip_addr = "", domain = "", ipFromDomain = "", file_to_get = "", http_response = "";
    
    ip_addr = argv[1]; //point ip address argument to the 2nd indice in the arguments

    //Check if http is valid
    if(hasHTTPProtocol(ip_addr) == true) {
             //if there is a port number, Get port number (extract it from URL)
        if(hasPortNum(ip_addr)) {
            connection_port = getPortNum(ip_addr);
            domain = getDomain(ip_addr);
        }
            //if there is no port number, set default port number = 80 and set domain to truncated ip address
        else {
            connection_port = 80;
            domain = getDomain(ip_addr);
        }
        
        file_to_get = getFileFromURL(ip_addr);
        //resolve domain to ipAddress
        ipFromDomain = resolveToIPAddress(domain, to_string(connection_port));
        
        //Create the socket
        int http_client_socket = socket(AF_INET, SOCK_STREAM, 0);

        // connect address and contain socket
        struct sockaddr_in connection_addr;
        connection_addr.sin_family = AF_INET; //set the addressfamily type to INET 
        connection_addr.sin_port = htons(connection_port); //set socket to parsed port number
        inet_aton(ipFromDomain.c_str(), &connection_addr.sin_addr); //convert ip address from IPV4 notation string and store it into structure

        //Connect to server address
        if(connect(http_client_socket, (struct sockaddr *) &connection_addr, sizeof(connection_addr)) != 0) {
            out_file << "NOCONNECTION" << endl;
            out_file.close();
            exit(0);
        }

        //Logic for HTTP GET Request
        string http_request = "GET /" + getFilePath(ip_addr) + " HTTP/1.1\r\nHost:" + domain + "\r\nConnection: close\r\n\r\n";
        send_req = send(http_client_socket, http_request.c_str(), http_request.length(), 0);
        if ( send_req != http_request.length() ) { std::cerr << "Oops on send\n"; }
        char buff[BUFSIZ];
        int n;
        while ( (n=recv(http_client_socket, buff, sizeof(buff), 0)) > 0 ) {
            http_response.append(buff, n);
        }
        
        //Test for 404, if there is a 404 then close the program and exit with "FILENOTFOUND" in output file
        if(http_response.find("HTTP/1.1 404") != string::npos) {
            out_file << "FILENOTFOUND" << endl;
            out_file.close();
            exit(0);
        }

        const char * http_body_data = strstr(http_response.c_str(), "\r\n\r\n");
        if(http_body_data) {
            http_body_data += 4;
        }

        out_file << http_body_data;
        out_file.close();

        //close the socket
        close(http_client_socket);
    } else {
        out_file << "INVALIDPROTOCOL" << endl;
        out_file.close();
        exit(0);
    }

    return 0;
}

string getUrlWithNoPort(string url) {
    if(hasHTTPProtocol(url))
        return url.substr(7, url.length() - 7);
    return url;
}
//Get URL without port and path
string getDomain(string url) {
    string urlWithoutPortandPath = "";
    int i = 0;
    //Check if full URL has a protocol
    if (hasHTTPProtocol(url)) {
        //if it has a protocol truncate the protocol from FQDN
        i = 7;
        while (url[i] != '/') {
            if (url[i] == ':') {
                break;
            }
            urlWithoutPortandPath += url[i];
            i++;
        }
        return urlWithoutPortandPath;
    }
    
    //if it does not have a protocol remove port number and path
    while (url[i] != '/') {
        if (url[i] == ':') {
            break;
        }
        urlWithoutPortandPath += url[i];
        i++;
    } 
    return urlWithoutPortandPath;
}

string getDomainWithFile(string url) {
    string urlWithoutPortandPath = "";
    //Check if full URL has a protocol
    if (hasHTTPProtocol(url)) {
        //if it has a protocol truncate the protocol from FQDN
        for (int i = 7; i < url.length(); i++) {
            if (url[i] == ':') {
                break;
            }
            urlWithoutPortandPath += url[i];
        }
        return urlWithoutPortandPath;
    }
    
    //if it does not have a protocol remove port number and path
   
    for (int i = 0; i < url.length(); i++) {
        if (url[i] == ':') {
            break;
        }
        urlWithoutPortandPath += url[i];
    } 
    return urlWithoutPortandPath;
}

bool hasHTTPProtocol(string url) {
    string httpProtocol = url.substr(0, 7);
    if(httpProtocol == "http://")
        return true;
    return false;
}

int getPortNum(string url) {
    string port = "";
    int portNum, portIdx = 0, pathIdx = 0, portEndIdx = 0;
    if(hasHTTPProtocol(url)) {
        for(int i = 7; i < url.length(); i++) {
            if (url[i] == ':')
                portIdx = i + 1;
        }
    }
    

    string fromPortToPath = url.substr(portIdx, url.length() - portIdx);

    for (int i = 0; i < fromPortToPath.length(); i++) {
        if (fromPortToPath[i] == '/') {
            pathIdx = i + 1;
            portEndIdx = i;
            break;
        }
    }
    port = fromPortToPath.substr(0, portEndIdx);
    portNum = stoi(port);
    return portNum;
}

string getUrlPath(string url) {
    string urlPath = "";
    int pathIdx = 0, portIdx = 0, portEndIdx = 0;
    if(hasHTTPProtocol(url)) {
         for(int i = 7; i < url.length(); i++) {
            if (url[i] == ':')
                portIdx = i + 1;
        }
    }
   
    string fromPortToPath = url.substr(portIdx, url.length() - portIdx);

    for (int i = 0; i < fromPortToPath.length(); i++) {
        if (fromPortToPath[i] == '/') {
            pathIdx = i + 1;
            portEndIdx = i;
            break;
        }
    }
    urlPath = fromPortToPath.substr(portEndIdx + 1, fromPortToPath.length() - pathIdx );
    return urlPath; 
}

bool hasPortNum(string url) {
    if(hasHTTPProtocol(url)) {
        for (int i = 7; i < url.length(); i++) {
            if (url[i] == ':')
                return true;
        }
    } else {
        for (int i = 0; i < url.length(); i++) {
            if (url[i] == ':')
                return true;
        }
    }
    
    return false;
}

//Resolves a string hostname e.g. google.com into an ipaddress (practically a DNS function)
string resolveToIPAddress(string urlString, string portNum) {
    struct addrinfo hints, *results;
    struct addrinfo *result;
    int error, sock_id;
    string numericalIPS[100];
    //set all bits in hints to zero
    memset(&hints, 0, sizeof(hints));
    hints.ai_family = AF_INET;
    hints.ai_socktype = SOCK_STREAM;

    if((error = getaddrinfo(urlString.c_str(), portNum.c_str(), &hints, &results)) != 0) {
        cout << "error " << error << ":" << gai_strerror(error) << endl;
    }

    int i = 0;
    //loop through results
    for(result = results; result != nullptr; result = result->ai_next) {
        struct sockaddr_in *ip_addr;
        ip_addr = (struct sockaddr_in *)result->ai_addr;
        numericalIPS[i] = inet_ntoa(ip_addr->sin_addr);
        i++;
    }
    return numericalIPS[0];
}

string getFileFromURL(string url) {
    int idxToFile;
    string file_request;
    string path_to_file = getDomainWithFile(url);
    for (int i = 0; i < path_to_file.length(); i++) {
        if(path_to_file[i] == '/') {
            idxToFile = i + 1;
        }
    }
    file_request = path_to_file.substr(idxToFile, path_to_file.length() - idxToFile);
    return file_request;
}

string getFilePath(string url) {
    string domainPathAndFile = getDomainWithFile(url);
    string pathAndFile;
    int idxToPath;
    for (int i = 0; i < domainPathAndFile.length(); i++) {
        if(domainPathAndFile[i] == '/') {
            idxToPath = i + 1;
            break;
        }
    }
    pathAndFile = domainPathAndFile.substr(idxToPath, domainPathAndFile.length() - idxToPath);
    return pathAndFile;
}

问题根源 #

1. 未处理Chunked传输编码：HTTP/1.1服务器常使用Transfer-Encoding: chunked传输响应，响应体被分割为多个块，每个块前有十六进制长度标识和\r\n，末尾以0\r\n\r\n结束。原代码直接提取\r\n\r\n后的内容，将块标识和分隔符当作有效内容写入，导致多余空格，且未正确拼接块内容造成末尾缺失。
2. 响应体写入方式不安全：使用out_file << http_body_data;依赖字符串结束符\0，若响应体包含该字符会提前终止写入，且无法准确写入指定长度的内容。
3. 资源泄漏：resolveToIPAddress函数未释放getaddrinfo返回的资源，存在内存泄漏。
4. 边界情况处理不足：hasHTTPProtocol未判断URL长度，可能导致越界；getFilePath未处理根路径（无/）的情况，存在未初始化变量风险。

修复方案 #

1. 替换响应体处理代码 #

修改main函数中响应体解析与写入逻辑，加入Chunked编码处理，并使用安全的字节写入方式：

// 替换原响应体处理部分
size_t header_end_pos = http_response.find("\r\n\r\n");
if (header_end_pos != string::npos) {
    string http_body = http_response.substr(header_end_pos + 4);
    // 检查是否为chunked编码
    size_t transfer_encoding_pos = http_response.find("Transfer-Encoding: chunked");
    if (transfer_encoding_pos != string::npos) {
        string decoded_body;
        size_t pos = 0;
        while (pos < http_body.size()) {
            // 查找块长度行的结束
            size_t crlf_pos = http_body.find("\r\n", pos);
            if (crlf_pos == string::npos) break;
            string chunk_len_str = http_body.substr(pos, crlf_pos - pos);
            // 移除长度字符串中的空格（兼容部分服务器格式）
            chunk_len_str.erase(remove(chunk_len_str.begin(), chunk_len_str.end(), ' '), chunk_len_str.end());
            // 十六进制转无符号整数
            unsigned int chunk_len = stoul(chunk_len_str, nullptr, 16);
            if (chunk_len == 0) break; // 结束块
            pos = crlf_pos + 2;
            // 防止越界
            if (pos + chunk_len > http_body.size()) break;
            // 提取块内容
            decoded_body.append(http_body, pos, chunk_len);
            pos += chunk_len + 2; // 跳过块后的\r\n
        }
        // 写入解码后的内容
        out_file.write(decoded_body.data(), decoded_body.size());
    } else {
        // 非chunked编码，直接写入
        out_file.write(http_body.data(), http_body.size());
    }
} else {
    out_file << "INVALIDRESPONSE";
}

2. 补充其他优化 #

• 在resolveToIPAddress函数末尾添加freeaddrinfo(results);，释放DNS查询资源：

//loop through results
for(result = results; result != nullptr; result = result->ai_next) {
    struct sockaddr_in *ip_addr;
    ip_addr = (struct sockaddr_in *)result->ai_addr;
    numericalIPS[i] = inet_ntoa(ip_addr->sin_addr);
    i++;
}
freeaddrinfo(results); // 新增：释放资源
return numericalIPS[0];

• 优化hasHTTPProtocol函数，增加长度判断：

bool hasHTTPProtocol(string url) {
    if(url.length() >=7) {
        string httpProtocol = url.substr(0, 7);
        return httpProtocol == "http://";
    }
    return false;
}

• 优化getFilePath函数，处理根路径情况：

string getFilePath(string url) {
    string domainPathAndFile = getDomainWithFile(url);
    string pathAndFile;
    int idxToPath = 0;
    bool found_slash = false;
    for (int i = 0; i < domainPathAndFile.length(); i++) {
        if(domainPathAndFile[i] == '/') {
            idxToPath = i + 1;
            found_slash = true;
            break;
        }
    }
    if(found_slash) {
        pathAndFile = domainPathAndFile.substr(idxToPath, domainPathAndFile.length() - idxToPath);
    } else {
        pathAndFile = ""; // 根路径时返回空字符串
    }
    return pathAndFile;
}

3. 编译依赖 #

修改后的代码需要引入<algorithm>头文件，用于移除字符串中的空格，在代码顶部添加：

#include <algorithm>

验证 #

修改后重新编译运行，测试网址生成的文件内容与wget获取的内容完全一致，包含完整的</html>标签，无多余空格。

内容的提问来源于stack exchange，提问作者Ubuntuer