You need to enable JavaScript to run this app.
优惠活动
大模型
产品
解决方案
定价
更多

C++流处理多字节字符异常:é等带重音字母无法正确识别

问题描述

我写了一段C++代码,功能是从一个.txt文件提取单词,再到另一个.txt文件中查找这些单词(匹配数量需在9-25之间),最后把结果保存到新的.txt文件里。

目前碰到多字节字符处理的问题:

  • 代码能正常识别あきる、會这类字符,但处理é这类带重音的字母时出问题,输出到.txt文件时é会变成e;
  • 把所有单词打印到控制台时,只能显示包含英文字符的单词。

我试过直接复制.txt文件能保留é,说明问题出在记录待写入字符位置的索引逻辑里。

相关代码
namespace fs = std::filesystem;
void processFile(const fs::path filePath, const fs::path resultPath, const std::unordered_set<std::wstring>& dict) {
    std::wifstream fileIn(filePath, std::ios::binary);

    // Create a locale using UTF-8 encoding
    std::locale utf8_locale(fileIn.getloc(), new std::codecvt_utf8<wchar_t>);

    fileIn.imbue(utf8_locale);

    std::vector<int> indexes;
    std::wstring previousWord;
    bool twelveInRow = false;
    int counter = 0;

    std::vector<std::wstring> words;
    std::wstring word;
    wchar_t prev_ch = L' ';
    wchar_t ch;

    while (fileIn.get(ch)) {
        ch = towlower(ch);

        if (iswalpha(ch)) {
            word += ch;
        }
        else if (iswspace(ch) && !iswspace(prev_ch) && !word.empty()) {
            words.push_back(std::move(word));
            word.clear();
        }
        prev_ch = ch;
    }

    if (!word.empty()) {
        words.push_back(std::move(word));
    }

    fileIn.close();

    for (int i = 0; i < words.size(); ++i) {
        std::wcout << words[i] << std::endl;
    }

    for (int i = 0; i < words.size(); ++i) {
        const auto& word = words[i];
        if (dict.find(word) == dict.end()) {
            if (previousWord != L"astralcomplex65") {
                indexes.push_back(-1);
                previousWord = L"astralcomplex65";
            }
        }
        else {
            indexes.push_back(i);
            previousWord = word;
            counter++;
            if (counter >= 9 && counter <= 25) {
                twelveInRow = true;
            }
        }
    }

    if (twelveInRow) {
        wchar_t prev_ch1 = L' ';
        wchar_t ch1;

        std::wifstream InFile(filePath, std::ios::binary);
        InFile.imbue(utf8_locale);
        //       std::filesystem::path filePath = std::filesystem::absolute(fileIn.getloc().name());
        std::wofstream file(resultPath / (randomName() + L"_Line_Matches" + L".txt"), std::ios::binary);
        // Set the locale for output using UTF-8 encoding
        file.imbue(utf8_locale);

        std::wstring content;
        content += filePath.wstring() + L'\n';
        for (auto i : indexes) {
            content += ((i == -1 ? L"" : words[i]) + L"\n");
        }
        content += L'\n';

        file << content;
        file.close();
        InFile.close();
    }
}

const std::unordered_set<std::wstring> loadDict(const fs::path path) {
    std::wifstream fileIn(path, std::ios::binary);

    // Create a locale using UTF-8 encoding
    std::locale utf8_locale(fileIn.getloc(), new std::codecvt_utf8<wchar_t>);

    fileIn.imbue(utf8_locale);

    std::unordered_set<std::wstring> words;
    std::wstring word;

    while (fileIn >> word) {
        words.insert(word);
    }

    fileIn.close();

    return words;
}

内容的提问来源于stack exchange,提问作者prt

相关产品推荐
方舟 Agent Plan

超全模态模型 × Harness 升级,最新支持 Deepseek-V4.1-Flash、GLM-5.3 系列、Doubao-Seedream-5.0-pro、Kimi-K3 (部分), 限时 9.9 元起

最近更新时间:2026.07.04 00:42:47