使用std::mbsrtowcs生成std::wstring触发std::length_error的问题求助

问题背景 #

测试通过不同方式读取文本文件（std::ifstream）并转换为std::wstring的耗时，实现了两种转换方法：

• 循环逐字符调用std::mbrtowc转换
• 调用std::mbsrtowcs一次性批量转换

待读取文本文件大小为3369045字节，预期转换后得到3324222个宽字符。逐字符转换运行正常，但批量转换触发如下错误：

terminate called after throwing an instance of 'std::length_error'
  what():  basic_string::_M_create
Aborted (core dumped)

相关代码 #

#include <iostream>
#include <fstream>
#include <filesystem>
#include <vector>
#include <cwchar>
#include <benchmark/benchmark.h>
#include <cassert>

static std::istream::pos_type get_file_size(std::istream& in)
{
    in.seekg(0, std::ios::end);
    auto pos = in.tellg();
    in.seekg(0, std::ios::beg);
    return pos;
}

std::wstring get_file_contents(const char* buffer, std::size_t size)
{
    std::mbstate_t state{};
    wchar_t current_wide_char;
    std::size_t remaining = size;
    std::size_t offset = 0;
    std::wstring output{};

    while (remaining > 0)
    {
        std::size_t len = std::mbrtowc(&current_wide_char, buffer + offset, remaining, &state);
        if (len == 0)
        {
            offset++;
            remaining--;
        }
        else if (len >= 1 && len <= remaining)
        {
            output += current_wide_char;
            offset += len;
            remaining -= len;
        }
        else if (len == static_cast<std::size_t>(-1))
        {
            std::cerr << "Failed: encoding error after offset " << offset << '\n';
            break;
        }
        else if (len == static_cast<std::size_t>(-2))
        {
            std::cerr << "Failed: incomplete sequence\n";
            break; 
        }
    }
    return output;
}

std::wstring get_file_contents2(const char* buffer, std::size_t size)
{
    std::mbstate_t state{};
    std::size_t len = std::mbsrtowcs(nullptr, &buffer, 0, &state);
    std::wstring output(len, L'\0');
    std::mbsrtowcs(output.data(), &buffer, len, &state);
    return output;
}

static void istreambuf_iterator(benchmark::State& state)
{
    std::setlocale(LC_ALL, "");
    for (auto _ : state)
    {
        std::ifstream in{"test.txt"};
        if (in.good())
        {
            std::vector<char> buffer{std::istreambuf_iterator<char>(in), std::istreambuf_iterator<char>()};
            const auto size = get_file_size(in);
            in.close();
            assert(buffer.size() == size);
            assert(size == 3369045u);

            const char* ptr = buffer.data();
            auto output = get_file_contents(ptr, size); 
            assert(output.length() == 3324222u);
            benchmark::DoNotOptimize(buffer);
            benchmark::DoNotOptimize(output);
        }
    }
}

static void istream_read_vec(benchmark::State& state)
{
    for (auto _ : state)
    {
        std::ifstream in{"test.txt"};
        if (in.good())
        {
            const auto size = get_file_size(in);
            std::vector<char> buffer(size, '\0');
            in.read(buffer.data(), size);
            in.close();
            assert(buffer.size() == size);
            assert(size == 3369045u);

            const char* ptr = buffer.data();
            auto output = get_file_contents(ptr, size);
            assert(output.length() == 3324222u);
            benchmark::DoNotOptimize(buffer);
            benchmark::DoNotOptimize(output);
        }
    }
}
static void istream_read_dynamic_arr(benchmark::State& state)
{
    for (auto _ : state)
    {
        std::ifstream in{"test.txt"};
        if (in.good())
        {
            const auto size = get_file_size(in);
            char* buffer = new char[size];
            in.read(buffer, size);
            in.close();
            assert(size == 3369045u);
            
            auto output = get_file_contents(buffer, size);
            
            assert(output.length() == 3324222u);
            delete[] buffer;
            benchmark::DoNotOptimize(buffer);
            benchmark::DoNotOptimize(output);
        }
    }
}
static void istream_read_string(benchmark::State& state)
{
    for (auto _ : state)
    {
        std::ifstream in{"test.txt"};
        if (in.good())
        {
            const auto size = get_file_size(in);
            std::string buffer(size, '\0');
            in.read(buffer.data(), size);
            in.close();
            assert(buffer.size() == size);
            assert(size == 3369045u);

            const char* ptr = buffer.c_str();
            auto output = get_file_contents(ptr, size);

            assert(output.length() == 3324222u);
            benchmark::DoNotOptimize(buffer);
            benchmark::DoNotOptimize(output);
        }
    }
}
BENCHMARK(istreambuf_iterator);
BENCHMARK(istream_read_vec);
BENCHMARK(istream_read_dynamic_arr);
BENCHMARK(istream_read_string);
BENCHMARK_MAIN();

将代码中的get_file_contents()替换为get_file_contents2()即可复现错误。

运行结果 #

逐字符转换（get_file_contents）输出 #

2025-03-12T12:11:19+05:30
Running ./benchmark
Run on (16 X 2304 MHz CPU s)
CPU Caches:
  L1 Data 32 KiB (x8)
  L1 Instruction 32 KiB (x8)
  L2 Unified 256 KiB (x8)
  L3 Unified 16384 KiB (x1)
Load Average: 0.20, 0.24, 0.18
-------------------------------------------------------------------
Benchmark                         Time             CPU   Iterations
-------------------------------------------------------------------
istreambuf_iterator        67161263 ns     67160588 ns            8
istream_read_vec           59514501 ns     59514555 ns           11
istream_read_dynamic_arr   63361615 ns     63299483 ns           12
istream_read_string        62134141 ns     62129082 ns           11

批量转换（get_file_contents2）输出 #

2025-03-12T12:12:11+05:30
Running ./benchmark
Run on (16 X 2304 MHz CPU s)
CPU Caches:
  L1 Data 32 KiB (x8)
  L1 Instruction 32 KiB (x8)
  L2 Unified 256 KiB (x8)
  L3 Unified 16384 KiB (x1)
Load Average: 0.11, 0.21, 0.17
-------------------------------------------------------------------
Benchmark                         Time             CPU   Iterations
-------------------------------------------------------------------
istreambuf_iterator        14109594 ns     14109764 ns           44
terminate called after throwing an instance of 'std::length_error'
  what():  basic_string::_M_create
Aborted (core dumped)

编译器版本：GCC 11.4.0

错误原因分析 #

get_file_contents2存在两个致命问题：

• 未限制多字节字符串处理长度，导致越界读取：第一次调用std::mbsrtowcs(nullptr, &buffer, 0, &state)时，第三个参数为0，函数会持续扫描直到遇到空字符。但文件读取的buffer没有末尾空字符，函数会越界读取内存，计算出异常大的len值，创建std::wstring时超过其最大容量，触发std::length_error。
• 指针被修改后未重置：mbsrtowcs会修改传入的src指针（即buffer），第一次调用后buffer已指向原始数据末尾，第二次调用无法正确读取内容。

修复后的代码 #

修改get_file_contents2如下：

std::wstring get_file_contents2(const char* buffer, std::size_t size)
{
    std::mbstate_t state{};
    const char* src = buffer; // 保存原始起始指针
    // 计算需要的宽字符数，限制处理的多字节字符长度为size
    std::size_t len = std::mbsrtowcs(nullptr, &src, size, &state);
    if (len == static_cast<std::size_t>(-1)) {
        std::cerr << "Failed: encoding error during length calculation\n";
        return {};
    }

    std::wstring output(len, L'\0');
    src = buffer; // 重置指针到起始位置
    std::mbstate_t state2{}; // 重置转换状态
    // 执行实际转换
    std::size_t converted = std::mbsrtowcs(output.data(), &src, len, &state2);
    if (converted != len) {
        std::cerr << "Warning: converted fewer characters than expected\n";
    }

    return output;
}

修复说明 #

• 保存原始buffer指针到src，避免被mbsrtowcs修改后丢失起始地址。
• 第一次调用mbsrtowcs时传入size作为第三个参数，限制处理的多字节字符长度，防止越界读取。
• 重置转换状态和指针后执行实际转换，确保从数据起始位置开始处理。
• 添加错误检查，处理编码错误或转换不完整的情况。

内容的提问来源于stack exchange，提问作者kiner_shah