使用Boost.Asio下载文件时输出文件存在冗余垃圾内容问题

AI 小北

2026-7-8

文件下载时开头结尾出现冗余内容的问题排查与修复

我尝试从服务器下载图片、HTML文本、压缩包等各类文件，但输出文件的开头和结尾总会出现冗余垃圾内容。以下是我的实现代码：

#include "dependencies.h"

asio::streambuf response_;
std::istream is(&response_);
std::ofstream file("file.html", std::ios::out);

void readHeader(asio::ssl::stream<tcp::socket>& socket);
void asyncRead(asio::ssl::stream<tcp::socket>& socket);

void readStatus(asio::ssl::stream<tcp::socket>& socket) {

    asio::async_read_until(socket, response_, "\r\n", [&](boost::system::error_code ec, size_t length) {
        if (!ec) {
            std::string header;
            is >> header;
            std::cout << "Protocol: " << header << '\n';
            std::string status_code;
            std::string status;
            is >> status_code;
            std::cout << "Status: " << status_code << '\n';
            std::string status_message;
            std::getline(is, status_message);

            readHeader(socket);
        }
        else if (ec != asio::error::eof) {
            std::cout << "[Status code] Error: " << ec.message() << '\n';
        }
    });
}

void readHeader(asio::ssl::stream<tcp::socket>& socket) {

    asio::async_read_until(socket, response_, "\r\n\r\n", [&](boost::system::error_code ec, size_t length) {
        if (!ec) {
            std::string status_message;
            while (getline(is, status_message) && status_message != "\r")
                std::cout << status_message << '\n';

            asyncRead(socket);
        }
        else if (ec != asio::error::eof) {
            std::cout << "[Header message] Error: " << ec.message() << '\n';
        }
        });
}

void asyncRead(asio::ssl::stream<tcp::socket>& socket) {
    asio::async_read(socket, response_, asio::transfer_at_least(1), [&](const boost::system::error_code ec, size_t length) {
        if (!ec) {
            file << &response_;

            asyncRead(socket);
        }
        else if (ec != asio::error::eof) {
            std::cout << "[Async reading] Error: " << ec.what() << std::endl;
        }
    });
}

int main(int args, const char* argv[]) {

    setlocale(LC_ALL, "Rus");

    boost::system::error_code ec;
    asio::io_context context;
    asio::ssl::context sslContext(asio::ssl::context::method::sslv23_client);
    sslContext.set_default_verify_paths();
    asio::ssl::stream<tcp::socket> socket(context, sslContext);

    //https://codeload.github.com/ERHZAN/NRM-Launcher/zip/refs/heads/main
    boost::urls::url url = boost::urls::url_view("https://en.cppreference.com/w/");
    std::string path = url.path();
    std::string host = url.host();
    std::string scheme = url.scheme();

    asio::streambuf request_;
    std::ostream os(&request_);
    os << "GET " << path << " HTTP/1.1" << "\r\n";
    os << "Host: " << host << "\r\n";
    os << "Accept: */*\r\n";
    os << "Connection: close\r\n\r\n";

    tcp::resolver resolver(context);
    tcp::resolver::query query(host, "https");
    resolver.async_resolve(query, [&](const boost::system::error_code& ec, tcp::resolver::iterator ep_iterator) {
        if (!ec) {
            socket.set_verify_mode(boost::asio::ssl::verify_peer);
            socket.set_verify_callback([&](bool preverified, boost::asio::ssl::verify_context& ctx) {
                char subject_name[256];
                X509* cert = X509_STORE_CTX_get_current_cert(ctx.native_handle());
                X509_NAME_oneline(X509_get_subject_name(cert), subject_name, 256);
                std::cout << "Verifying " << subject_name << "\r\n";

                return true;
            });

            socket.lowest_layer().async_connect(ep_iterator->endpoint(), [&](const boost::system::error_code& ec) {
                if (!ec) {
                    socket.async_handshake(asio::ssl::stream_base::handshake_type::client, [&](const boost::system::error_code& ec) {
                        if (!ec) {
                            asio::async_write(socket, request_, [&](const boost::system::error_code ec, size_t length) {
                                if (!ec) {
                                    readStatus(socket);
                                }
                                else {
                                    std::cout << ec.what() << std::endl;
                                }
                            });
                        }
                        else {
                            std::cout << ec.what() << std::endl;
                        }
                    });
                }
                else {
                    std::cout << "[Connection] Error: " << ec.what() << std::endl;
                }
                });
        }
    });
    context.run();

    return 0;
}

输出文件示例：

ab5b

<!DOCTYPE html>
<html lang="en" dir="ltr" class="client-nojs">
我已为便于阅读删除了部分HTML内容
</html>


0

问题核心原因

分块编码未解析：服务器返回的是Transfer-Encoding: chunked格式响应，代码直接将分块的长度标识（如示例中的ab5b）和结束标记（0）写入了文件，这是冗余内容的主要来源。
streambuf残留数据：读取响应头后，缓冲区中残留的未处理字节（如换行符、回车符）被后续写入操作带入文件，导致开头冗余。
文件写入方式错误：直接使用file << &response_会输出整个缓冲区内容，包括未消耗的残留数据，且循环读取结束后可能写入空数据，造成结尾冗余。

修复方案

1. 解析分块编码响应

添加分块解析逻辑，读取分块长度后再读取对应的数据，跳过分块标记：

void readChunkSize(asio::ssl::stream<tcp::socket>& socket);

void asyncReadChunk(asio::ssl::stream<tcp::socket>& socket, std::size_t chunkSize) {
    asio::async_read(socket, response_, asio::transfer_exactly(chunkSize), [&](const boost::system::error_code ec, size_t length) {
        if (!ec) {
            // 写入分块数据
            file.write(asio::buffer_cast<const char*>(response_.data()), length);
            response_.consume(length);
            
            // 跳过分块后的换行符
            asio::async_read_until(socket, response_, "\r\n", [&](const boost::system::error_code ec, size_t) {
                if (!ec) {
                    response_.consume(response_.size());
                    readChunkSize(socket);
                }
            });
        }
        else if (ec != asio::error::eof) {
            std::cout << "[Async reading chunk] Error: " << ec.what() << std::endl;
        }
    });
}

void readChunkSize(asio::ssl::stream<tcp::socket>& socket) {
    asio::async_read_until(socket, response_, "\r\n", [&](const boost::system::error_code ec, size_t length) {
        if (!ec) {
            std::string chunkSizeStr;
            std::getline(is, chunkSizeStr);
            if (!chunkSizeStr.empty() && chunkSizeStr.back() == '\r') {
                chunkSizeStr.pop_back();
            }
            
            std::size_t chunkSize = std::stoul(chunkSizeStr, nullptr, 16);
            if (chunkSize == 0) {
                // 分块结束，跳过最后的换行符
                asio::async_read_until(socket, response_, "\r\n", [&](const boost::system::error_code ec, size_t) {
                    if (!ec) {
                        response_.consume(response_.size());
                        file.close();
                    }
                });
                return;
            }
            
            response_.consume(response_.size());
            asyncReadChunk(socket, chunkSize);
        }
        else if (ec != asio::error::eof) {
            std::cout << "[Reading chunk size] Error: " << ec.what() << std::endl;
        }
    });
}

2. 清空缓冲区残留数据

在读取完响应头后，清空缓冲区，确保后续只读取响应体：

void readHeader(asio::ssl::stream<tcp::socket>& socket) {
    asio::async_read_until(socket, response_, "\r\n\r\n", [&](boost::system::error_code ec, size_t length) {
        if (!ec) {
            std::string status_message;
            std::string transferEncoding;
            while (getline(is, status_message) && status_message != "\r") {
                std::cout << status_message << '\n';
                // 检查是否为分块编码
                if (status_message.substr(0, 17) == "Transfer-Encoding:") {
                    transferEncoding = status_message.substr(18);
                    transferEncoding.erase(0, transferEncoding.find_first_not_of(" \t"));
                }
            }
            
            // 清空缓冲区残留数据
            response_.consume(response_.size());
            
            // 根据编码类型选择读取方式
            if (transferEncoding == "chunked") {
                readChunkSize(socket);
            } else {
                // 这里可补充Content-Length解析逻辑，读取指定长度数据
                asyncRead(socket);
            }
        }
        else if (ec != asio::error::eof) {
            std::cout << "[Header message] Error: " << ec.message() << '\n';
        }
    });
}

3. 改进文件写入逻辑

避免直接写入streambuf指针，按实际读取长度写入并消耗缓冲区：

void asyncRead(asio::ssl::stream<tcp::socket>& socket) {
    asio::async_read(socket, response_, asio::transfer_at_least(1), [&](const boost::system::error_code ec, size_t length) {
        if (!ec) {
            file.write(asio::buffer_cast<const char*>(response_.data()), length);
            response_.consume(length);
            asyncRead(socket);
        }
        else if (ec != asio::error::eof) {
            std::cout << "[Async reading] Error: " << ec.what() << std::endl;
        } else {
            file.close();
        }
    });
}

4. 二进制模式打开文件

下载二进制文件（图片、压缩包）时，用二进制模式避免换行符转换损坏文件：

std::ofstream file("file.html", std::ios::out | std::ios::binary);

内容的提问来源于stack exchange，提问作者bergemon