C++模式Bison/Flex编译器后端解析器遇Flex特定输入报错

问题现象 #

使用C++模式的Bison和Flex开发编译器后端解析器时，多数场景运行正常，但特定输入会触发Flex致命错误：fatal flex scanner internal error--end of buffer missed。只要对输入文件做任意字符的增删改操作，扫描器就能恢复正常工作。

扫描器代码 #

%{
#include <string>

#include "lexer.hpp"
using Token = yy::Parser::token;

#define yyterminate() return( Token::YYEOF )

#define YY_USER_ACTION yylloc->step(); yylloc->columns(yyleng);
#define YY_VERBOSE

%}

/* Target the C++ implementation */
%option c++
/* Leave buffer-switching to us */
%option noyywrap 
/* Don't generate a default rule on our behalf */
%option nodefault
/* Don't try to #include unistd */
%option nounistd

/* Don't try to push tokens back into the input stream */
%option noinput 
%option nounput
%option stack
%option verbose
%option noyylineno
%option noyyget_lineno yyset_lineno yyget_out yyset_out yyget_in yyset_in
%option warn
%option noyymore
%option ecs
%option align
%option read
/* We're not writing an interpreter */
%option never-interactive batch 
/* Write a source file, but not a header file */
%option outfile="lexer.cpp"

%{

int yyFlexLexer::yylex() { abort(); }

bool Lexer::setBuffer(std::string_view str) noexcept
{
    yy_buffer_state* state = static_cast<yy_buffer_state*>(yyalloc(sizeof(yy_buffer_state)));
    if (!state)
    {
        return (false);
    }
    memset(state, 0, sizeof(yy_buffer_state));

    state->yy_buf_size = (int) (str.size());
    state->yy_buf_pos = state->yy_ch_buf = const_cast<char*>(str.data());
    state->yy_is_our_buffer = 0;
    state->yy_input_file = NULL;
    state->yy_n_chars = state->yy_buf_size;
    state->yy_is_interactive = 0;
    state->yy_at_bol = 1;
    state->yy_fill_buffer = 0;
    state->yy_buffer_status = YY_BUFFER_NEW;

    yy_switch_to_buffer( state );

    return (true);
}

%}


/* ---- Named pattern fragments ------------------------------------------- */
newline     (\n)


%x POST_SIGIL

/* vvvv Comments not allowed beyond this point vvvvvvvvvvvvvvvvvvvvvvvvvvvv */
%%

%{
    yylloc->step();
%}

<POST_SIGIL>[a-zA-Z][a-zA-Z0-9_]* {
        yylval->emplace<IdentifierToken>(yytext);
        BEGIN(INITIAL);
        return Token::IDENTIFIER;
    }

<POST_SIGIL>. { throw yy::Parser::syntax_error(*yylloc, "invalid character: " + std::string(yytext)); }

<INITIAL>{
"export"    return Token::EXPORT;
"thread"    return Token::THREAD;
"section"   return Token::SECTION;

"env"   return Token::ENV;
"phi"   return Token::PHI;

"type"  return Token::TYPE;
"align" return Token::ALIGN;

"data"  return Token::DATA;

"$"         BEGIN(POST_SIGIL); return Token::DOLLAR;
","         return Token::COMMA;
"..."       return Token::ELLIPSIS;
"@"         BEGIN(POST_SIGIL); return Token::AT;
"%"         BEGIN(POST_SIGIL); return Token::PERCENT;
"function"  return Token::FUNCTION;
"("         return Token::LPAREN;
")"         return Token::RPAREN;
"{"         return Token::LBRACE;
"}"         return Token::RBRACE;
"\+"        return Token::PLUS;
":"         BEGIN(POST_SIGIL); return Token::COLON;

"sb"        return Token::SB;
"ub"        return Token::UB;
"sh"        return Token::SH;
"uh"        return Token::UH;

"jmp"       return Token::JMP;
"jnz"       return Token::JNZ;
"hlt"       return Token::HLT;
"ret"       return Token::RET;
"call"      return Token::CALL;
"cast"      return Token::CAST;
"copy"      return Token::COPY;

"="         return Token::EQUALS;

"w" return Token::W;
"l" return Token::L;
"s" return Token::S;
"d" return Token::D;
"b" return Token::B;
"h" return Token::H;
"z" return Token::Z;

store(d|s|l|w|h|b) {
    yylval->emplace<StoreToken>(yytext[5]);
    return Token::STORE; 
}

load {
    yylval->emplace<LoadToken>("d"); //FIXME
    return Token::LOAD; 
}

load(d|s|l|w|h|b|sw|uw|sh|uh|sb|ub) { 
    yylval->emplace<LoadToken>(yytext + 4); 
    return Token::LOAD; 
}

"blit" return Token::BLIT;

alloc(4|8|16) { 
    yylval->emplace<AllocToken>(static_cast<int8_t>(std::stoi(yytext + 5))); 
    return Token::ALLOC; 
}

c(sle|slt|sge|sgt|ule|ult|uge|ugt)(w|l|s|d|q|t|h|f) {
    yylval->emplace<CompareToken>(std::string{yytext + 1, yytext + 4}, yytext + 4);
    return Token::COMPARE; 
}

c(eq|ne|le|lt|ge|gt|uo)(w|l|s|d|q|t|h|f) {
    yylval->emplace<CompareToken>(std::string{yytext + 1, yytext + 3}, yytext + 3);
    return Token::COMPARE; 
}

co(w|l|s|d|q|t|h|f) {
    yylval->emplace<CompareToken>("o", yytext + 2);
    return Token::COMPARE; 
}

(extsw|extuw|extsh|extuh|extsb|extub|exts|truncd|stosi|stoui|dtosi|dtoui|swtof|uwtof|sltof|ultof) {
    yylval->emplace<ConversionToken>();
    return Token::CONVERSION;
}

(add|and|div|mul|neg|or|rem|sar|shl|shr|sub|udix|urem|xor) {
    yylval->emplace<BinaryToken>();
    return Token::BINARY;
}

[-]?[0-9]+ {
    yylval->emplace<uint64_t>(std::stoll(yytext));
    return Token::NUMBER;
}

(s_|d_)?[-]?[0-9]*\.[0-9]+ {
    if(yytext[0] == 's' || yytext[0] == 'd')
        yylval->emplace<double>(std::stod(yytext + 2));
    else
        yylval->emplace<double>(std::stod(yytext));

    return Token::FLOAT;
}

\"[^\"]*\" {
    yylval->emplace<std::string>(yytext + 1, yyleng - 2);
    return Token::STRING_LITERAL;
}

"#".*       yylloc->step();
[ \t]+      yylloc->step();
{newline}+  yylloc->lines (yyleng); yylloc->step();

[a-zA-Z][a-zA-Z0-9_]* {
    yylval->emplace<IdentifierToken>(yytext + 1); 
    return Token::IDENTIFIER;
}

.   { throw yy::Parser::syntax_error(*yylloc, "invalid character: " + std::string(yytext)); }

<<EOF>> return Token::YYEOF;
}

触发错误的输入文件 #

（注意末尾的空行）

# tests that the address matcher is not
# confused by the two multiplications

# note: the code handling apple asm fixes
# ruins the good work of the matcher here,
# I should revisit these fixes

export function w $f(l %i, l %j) {
@start
    %off1 =l mul %i, 8
    %a_i =l add $a, %off1
    %off2 =l mul %j, 4
    %a_ij =l add %a_i, %off2
    %x =w loadsw %a_ij
    ret %x
}

# >>> driver
# int a[] = {1, 2, 3, 4};
# extern int f(long long, long long);
# int main() {
#   return !(f(0, 0) == 1 && f(0, 1) == 2 && f(1, 0) == 3 && f(1, 1) == 4);
# }
# <<<

问题根源与修复方案 #

核心原因 #

错误的本质是手动构造yy_buffer_state时未遵守Flex的缓冲区规范：Flex要求缓冲区末尾必须有至少两个连续的NUL字节（\0），用于处理匹配时的边界检查。你的setBuffer方法直接使用输入字符串的原始数据，而std::string_view可能没有NUL终止，std::string也只有一个末尾NUL，导致Flex扫描到缓冲区末尾时无法正确识别EOF，触发内部错误。

此外还有两个次要问题：

1. 标识符解析规则错误：[a-zA-Z][a-zA-Z0-9_]*规则中错误地使用yytext + 1，会丢失标识符的第一个字符。
2. 冗余的<<EOF>>规则：yyterminate()宏已经处理了EOF返回，重复规则会导致逻辑冲突。

修复步骤 #

1. 正确构造Flex缓冲区

推荐使用Flex提供的yy_scan_bytes函数自动处理缓冲区，避免手动构造的错误：

bool Lexer::setBuffer(std::string_view str) noexcept
{
    // 先清理当前缓冲区防止内存泄漏
    if (YY_CURRENT_BUFFER) {
        yy_delete_buffer(YY_CURRENT_BUFFER);
    }
    // 使用yy_scan_bytes处理任意字节序列，无需手动添加NUL
    yy_buffer_state* state = yy_scan_bytes(str.data(), static_cast<int>(str.size()));
    if (!state) {
        return false;
    }
    yy_switch_to_buffer(state);
    return true;
}

如果必须手动构造缓冲区，需确保末尾有两个NUL字节：

bool Lexer::setBuffer(std::string_view str) noexcept
{
    const size_t buf_size = str.size() + 2;
    char* buf = static_cast<char*>(yyalloc(buf_size));
    if (!buf) {
        return false;
    }
    memcpy(buf, str.data(), str.size());
    buf[str.size()] = '\0';
    buf[str.size() + 1] = '\0';

    yy_buffer_state* state = static_cast<yy_buffer_state*>(yyalloc(sizeof(yy_buffer_state)));
    if (!state) {
        yyfree(buf);
        return false;
    }
    memset(state, 0, sizeof(yy_buffer_state));

    state->yy_buf_size = static_cast<int>(buf_size);
    state->yy_buf_pos = state->yy_ch_buf = buf;
    state->yy_is_our_buffer = 1; // 标记为Flex可自动释放的缓冲区
    state->yy_input_file = NULL;
    state->yy_n_chars = static_cast<int>(str.size());
    state->yy_is_interactive = 0;
    state->yy_at_bol = 1;
    state->yy_fill_buffer = 0;
    state->yy_buffer_status = YY_BUFFER_NEW;

    yy_switch_to_buffer(state);
    return true;
}

2. 修复标识符解析错误

将标识符规则中的yytext + 1改为yytext：

[a-zA-Z][a-zA-Z0-9_]* {
    yylval->emplace<IdentifierToken>(yytext); 
    return Token::IDENTIFIER;
}

3. 移除冗余的EOF规则

删除<<EOF>> return Token::YYEOF;，依赖yyterminate()宏处理EOF即可。

内容的提问来源于stack exchange，提问作者Frederic Schönberger