Lex中yytext累积单词问题：词频统计程序异常求助

我编写了一个读取单个或多个文件、统计单词出现次数及位置的程序，目前遇到yytext累积单词的问题。预期效果应为类似cat line 5 line 4 file 1的形式，但程序甚至无法进入用于比较符号表中已有单词与当前单词的if条件。实际出现单词累积的情况：比如"cat owns"本应是cat和owns分别存入不同节点，使用strcmp进行比较，但现在却是用"owns"与"cat owns"进行比较。

程序代码 #

%option noyywrap
%option nodefault
%option yylineno
%option warn
%option case-insensitive

%{
  #include <stdlib.h>
  #include <stdio.h>
  #include <string.h>

  typedef struct _la{      /* no da lista de ocorrencias */
    int line;
    char * arq;
    struct _la * next;
  } LineArq;

  typedef struct _node {    /* entrada na tabela de simbolos */
   char * word;
   LineArq *lines;
   struct _node * next;
  } Node;


  char *curfilename;

  Node * tabela = NULL;

  Node * getNodeL( char * word, int line, char * arq );
  void insertNode(char * word, int line, char * arq, Node **l);
  void show(Node *l);
%}

%%

a |
an |
and |
are |
as |
at |
be |
but |
for |
in |
is |
it |
of |
on |
or |
that |
the |
this |
to               { /* printf("DEBUG: [%d, %s] %s\n", yylineno, curfilename, yytext ); */ }  /* ESSES PADROES SAO IGNORADOS... */

[a-z]+(\'(s|t))?     { insertNode(yytext, yylineno, curfilename, &tabela);
                    /* printf("DEBUG: [%d, %s] %s\n", yylineno, curfilename, yytext ); */
                 }

.|\n             { /* printf("DEBUG: [%d, %s] %s\n", yylineno, curfilename, yytext ); */ }  /* IGNORA NOVA LINHA E QUALQUER OUTRO CARACTER... */

%%

int main(int argc, char *argv[]){
  int i = 0;

  for(i = 1; i < argc; i++) {
    FILE *f = fopen(argv[i], "r");
    if(!f) {
      perror(argv[i]);
      return (1);
    }
    curfilename = argv[i]; 
    yyrestart(f);
    yylineno = 1;
    yylex();
    fclose(f);
  }

  show(tabela);
}

Node * getNodeL(char * word, int line, char * arq){
  
  Node * tmp = (Node *) malloc(sizeof(Node));

  if ( tmp == NULL ) {
    printf("Erro ao alocar memoria para o no da lista\n");
    exit( 1023 ); /* documentar no Dicionario de dados */
  }

  LineArq * la = (LineArq *) malloc(sizeof(LineArq));

  if (la == NULL){
    printf("Erro ao alocar memoria para o no da lista\n");
    exit( 1024 ); /* documentar no Dicionario de dados */
  }

  la->line = line;
  la->arq = arq;
  la->next = NULL;
  
  tmp->word = word;
  tmp->lines = la;
  tmp->next = NULL;

  return tmp;
}

void insertNode(char * word, int line, char * arq, Node **l){


  printf("[DEBUG] >>>>>>>> %s     %d 	 %s\n", word, line, arq);

  Node * p = *l;
  int count = 0;
  
  /* primeira entrada na tabela */
  if ( p == NULL ) {
    printf("entrou\n");
    *l = getNodeL(word, line, arq);
    return;
  }
  printf("%s\n", p->word);

  while( p != NULL ) {
     printf("[DEBUG]       [%s] == [%s]\n", word, p->word);
     if ( strcmp(word, p->word) == 0 ) {
        printf("entrou palavra igual\n");
        break;
     }
     p = p-> next;
  }

  if ( p != NULL ) { /* palavra encontrada na tabela */
      LineArq * la = (LineArq *) malloc(sizeof(LineArq));
      la->line = line;
      la->arq = arq;
      p->lines->next = la;
      la->next = NULL;
  } else {
    Node * new = getNodeL(word, line, arq);
    new->next = *l;
    *l = new;
  }
}

void show(Node *l){
  Node * tmp = l;
  LineArq * linesA;
  while( tmp != NULL ) {
    printf("Palavra: %10s\n", tmp->word);
    linesA = tmp->lines;
    while (linesA != NULL){
      printf("	Linha: %d 	 Arquivo: %s\n", linesA->line, linesA->arq);
      linesA = linesA->next;
    }
    /*
    if ( tmp->next != NULL ) {
        printf(", ");
    }
    */
    tmp = tmp->next;
  }
}

程序输出结果 #

because the cat prefers]
[DEBUG] >>>>>>>> colder 3 teste.txt
prefers colder
[DEBUG] [colder] == [prefers colder]
[DEBUG] [colder] == [cat prefers colder]
[DEBUG] [colder] == [because the cat prefers colder]
[DEBUG] [colder] == [water]
because the cat prefers colder]
[DEBUG] [colder] = [warm water
because the cat prefers colder]
[DEBUG] [colder] == [drink warm water
because the cat prefers colder]
[DEBUG] [colder] == [like to drink warm water
because the cat prefers colder]
[DEBUG] [colder] = [doesn't like to drink warm water]
because the cat prefers colder]
[DEBUG] [colder] = [cat doesn't like to drink warm water]
because the cat prefers colder]
[DEBUG] [colder] == [owner
The cat doesn't like to drink warm water
because the cat prefers colder]
[DEBUG] [colder] = [its owner
The cat doesn't like to drink warm water
because the cat prefers colder]
[DEBUG] [colder] = [own's its owner
The cat doesn't like to drink warm water
because the cat prefers colder]
[DEBUG] [colder] = [cat own's it’s owner

问题分析与修复 #

核心问题 #

你直接将Flex的yytext指针赋值给结构体的word字段，但yytext是Flex内部输入缓冲区的临时指针，后续扫描操作会修改这个缓冲区的内容（包括扩容、覆盖原有数据），导致之前保存的节点里的word指向的内容被篡改，出现单词累积、拼接的异常情况。

修复方法 #

必须为每个单词分配独立的内存，保存yytext的副本，而不是直接复用指针。修改getNodeL函数中的单词赋值逻辑：

Node * getNodeL(char * word, int line, char * arq){
  
  Node * tmp = (Node *) malloc(sizeof(Node));
  if ( tmp == NULL ) {
    printf("Erro ao alocar memoria para o no da lista\n");
    exit( 1023 );
  }

  LineArq * la = (LineArq *) malloc(sizeof(LineArq));
  if (la == NULL){
    printf("Erro ao alocar memoria para o no da lista\n");
    exit( 1024 );
  }

  la->line = line;
  la->arq = arq; // 此处curfilename指向argv参数，程序运行期间有效，若需更安全可改为strdup(arq)
  la->next = NULL;
  
  // 复制单词到独立内存
  tmp->word = strdup(word);
  if (tmp->word == NULL) {
    printf("Erro ao alocar memoria para a palavra\n");
    free(tmp);
    free(la);
    exit(1025);
  }
  tmp->lines = la;
  tmp->next = NULL;

  return tmp;
}

额外优化（可选） #

为避免内存泄漏，可添加一个释放符号表内存的函数，并在程序结束时调用：

void freeTable(Node **l) {
  Node *tmp = *l;
  while (tmp != NULL) {
    Node *nextNode = tmp->next;
    // 释放行号与文件链表
    LineArq *laTmp = tmp->lines;
    while (laTmp != NULL) {
      LineArq *nextLa = laTmp->next;
      // 如果la->arq用了strdup，此处需添加free(laTmp->arq);
      free(laTmp);
      laTmp = nextLa;
    }
    // 释放单词内存
    free(tmp->word);
    free(tmp);
    tmp = nextNode;
  }
  *l = NULL;
}

在main函数的show(tabela);之后添加：

freeTable(&tabela);

内容的提问来源于stack exchange，提问作者PRYSCILLA ROMÃO DE MEDEIROS