Intel PIN双缓冲指令追踪工具SIGSEGV错误排查求助

问题背景 #

我编写了一个用于指令追踪的Intel PIN工具，通过单独线程将缓冲区内容写入文件。运行时多数情况下触发SIGSEGV错误，偶尔能正常执行，错误信息如下：

➤ $PIN_ROOT/pin -t obj-intel64/chvp_tracer.so -- ./a.out                                                                                                                      (base) 
C: [tid:49808] Tool (or Pin) caused signal 11 at PC 0x78dd8c72a238
fish: Job 1, '$PIN_ROOT/pin -t obj-intel64/ch…' terminated by signal SIGSEGV (Address boundary error)

问题定位 #

代码中的双缓冲线程同步存在多个线程安全问题，这是导致SIGSEGV的核心原因：

• dump_first_buffer变量无保护访问：该变量是双缓冲切换的核心标志，但在多线程读写时仅部分场景加锁，存在竞态条件，可能导致访问错误的缓冲区索引引发内存越界。
• 信号量使用逻辑错误：写入线程先等待信号量再加锁，可能唤醒后缓冲区索引已被修改；且退出逻辑中的信号量等待存在死锁风险。
• 锁顺序与缓冲区切换问题：写入逻辑中先持有写锁再获取读锁，可能引发死锁；切换缓冲区时未确保新写入缓冲区处于空闲状态。
• 全局变量curr_instr无保护：直接使用未加锁的全局变量curr_instr，多线程写入会导致数据竞争，造成缓冲区指令数据损坏。

修复后的代码 #

核心修改点 #

• 对dump_first_buffer的所有读写操作加锁保护
• 调整信号量逻辑，确保缓冲区切换后再唤醒写入线程
• 修正锁持有顺序，避免死锁
• 新增锁保护curr_instr，避免数据竞争
• 优化退出逻辑，确保剩余缓冲区数据被处理

bool instrumentation_done = false;

constexpr uint32_t buffer_size = 2048;
std::array<std::vector<trace_instr_format_t>,2> double_buffer;
bool dump_first_buffer=false;

PIN_MUTEX buffer_locks[2];
PIN_MUTEX curr_instr_lock; // 新增：保护全局变量curr_instr
PIN_SEMAPHORE sm;

#define READ_LOCK 0
#define WRITE_LOCK 1

void WriteCurrentInstruction()
{
  if(last_instr!=first_instr and instrCount > last_instr)
    PIN_ExitApplication(0);

  // 先拷贝curr_instr到局部变量，避免多线程数据竞争
  PIN_MutexLock(&curr_instr_lock);
  trace_instr_format_t local_instr = curr_instr;
  PIN_MutexUnlock(&curr_instr_lock);

  PIN_MutexLock(&buffer_locks[WRITE_LOCK]);
  int32_t idx = dump_first_buffer;
  auto &buff=double_buffer[idx];
  if(buff.size()>=buffer_size)
  {
    // 获取读锁保护dump_first_buffer的修改
    PIN_MutexLock(&buffer_locks[READ_LOCK]);
    dump_first_buffer = !dump_first_buffer;
    idx = dump_first_buffer;
    // 唤醒写入线程处理填满的缓冲区
    PIN_SemaphoreSet(&sm);
    PIN_MutexUnlock(&buffer_locks[READ_LOCK]);
    
    // 校验新写入缓冲区是否为空，避免溢出
    if(double_buffer[idx].size() > 0) {
      LOG("Error: new write buffer is not empty!");
    }
  }
  buff.push_back(local_instr);
  PIN_MutexUnlock(&buffer_locks[WRITE_LOCK]);
}


VOID ActuallyWriteInstruction(VOID * args)
{
  uint64_t insts=0;
  while(!instrumentation_done)
  {
    PIN_SemaphoreWait(&sm);
    PIN_MutexLock(&buffer_locks[READ_LOCK]);
    // 获取待写入的缓冲区索引（刚填满的那个）
    int32_t idx = !dump_first_buffer;
    auto &buff=double_buffer[idx];
    if(buff.empty()) {
      PIN_MutexUnlock(&buffer_locks[READ_LOCK]);
      continue;
    }
    // 释放信号量，允许主线程继续切换缓冲区
    PIN_SemaphoreClear(&sm);
    PIN_MutexUnlock(&buffer_locks[READ_LOCK]);

    // 写入文件操作，此时缓冲区已被独占
    for(const auto &instr:buff)
    {
      outfile.write(reinterpret_cast<const char*>(&instr.ip),sizeof(instr.ip));
      outfile.write(reinterpret_cast<const char*>(&instr.inst_class),sizeof(instr.inst_class));
      if(instr.is_branch())
      {
        outfile.write(reinterpret_cast<const char*>(&instr.branch_taken),sizeof(instr.branch_taken));
        outfile.write(reinterpret_cast<const char*>(&instr.branch_target),sizeof(instr.branch_target));
      }
      outfile.write(reinterpret_cast<const char*>(&instr.num_operands),sizeof(instr.num_operands));
      for(const auto &op:instr.operands)
      {
        outfile.write(reinterpret_cast<const char*>(&op.header),sizeof(op.header));
        outfile.write(reinterpret_cast<const char*>(&op.element_size),sizeof(op.element_size));
        outfile.write(reinterpret_cast<const char*>(&op.num_elements),sizeof(op.num_elements));
        outfile.write(reinterpret_cast<const char*>(&op.base_reg),sizeof(op.base_reg));
        if(op.isMem())
        {
          outfile.write(reinterpret_cast<const char*>(&op.index_reg),sizeof(op.index_reg));
          outfile.write(reinterpret_cast<const char*>(&op.scale),sizeof(op.scale));
          outfile.write(reinterpret_cast<const char*>(&op.disp),sizeof(op.disp));
          outfile.write(reinterpret_cast<const char*>(&op.addr),sizeof(op.addr));
        }
        if(op.is_write())
        {
          const auto num_bytes = op.element_size*op.num_elements;
          outfile.write(reinterpret_cast<const char*>(&op.value),num_bytes);
        }
      }
    }

    // 清理已写入的缓冲区
    PIN_MutexLock(&buffer_locks[READ_LOCK]);
    insts+=buff.size();
    buff.clear();
    LOG("Instructions done "+std::to_string(insts)+"\n");
    PIN_MutexUnlock(&buffer_locks[READ_LOCK]);
  }

  // 处理程序结束后剩余的缓冲区数据
  PIN_MutexLock(&buffer_locks[READ_LOCK]);
  int32_t idx = !dump_first_buffer;
  auto &buff=double_buffer[idx];
  if(!buff.empty()) {
    for(const auto &instr:buff)
    {
      outfile.write(reinterpret_cast<const char*>(&instr.ip),sizeof(instr.ip));
      outfile.write(reinterpret_cast<const char*>(&instr.inst_class),sizeof(instr.inst_class));
      if(instr.is_branch())
      {
        outfile.write(reinterpret_cast<const char*>(&instr.branch_taken),sizeof(instr.branch_taken));
        outfile.write(reinterpret_cast<const char*>(&instr.branch_target),sizeof(instr.branch_target));
      }
      outfile.write(reinterpret_cast<const char*>(&instr.num_operands),sizeof(instr.num_operands));
      for(const auto &op:instr.operands)
      {
        outfile.write(reinterpret_cast<const char*>(&op.header),sizeof(op.header));
        outfile.write(reinterpret_cast<const char*>(&op.element_size),sizeof(op.element_size));
        outfile.write(reinterpret_cast<const char*>(&op.num_elements),sizeof(op.num_elements));
        outfile.write(reinterpret_cast<const char*>(&op.base_reg),sizeof(op.base_reg));
        if(op.isMem())
        {
          outfile.write(reinterpret_cast<const char*>(&op.index_reg),sizeof(op.index_reg));
          outfile.write(reinterpret_cast<const char*>(&op.scale),sizeof(op.scale));
          outfile.write(reinterpret_cast<const char*>(&op.disp),sizeof(op.disp));
          outfile.write(reinterpret_cast<const char*>(&op.addr),sizeof(op.addr));
        }
        if(op.is_write())
        {
          const auto num_bytes = op.element_size*op.num_elements;
          outfile.write(reinterpret_cast<const char*>(&op.value),num_bytes);
        }
      }
    }
    buff.clear();
  }
  PIN_MutexUnlock(&buffer_locks[READ_LOCK]);
}

VOID Fini(INT32 code, VOID* v) {
  LOG("Done, waiting for IO \n");
  // 通知写入线程结束
  instrumentation_done = true;
  // 唤醒写入线程，确保它能退出循环
  PIN_SemaphoreSet(&sm);
  PIN_WaitForThreadTermination(write_thid,PIN_INFINITE_TIMEOUT,NULL);
  
  LOG("Done, Exiting \n");
  PIN_MutexFini(&buffer_locks[0]);
  PIN_MutexFini(&buffer_locks[1]);
  PIN_MutexFini(&curr_instr_lock);
  PIN_SemaphoreFini(&sm);
  outfile.close();
}

int main(int argc, char* argv[])
{
  if (PIN_Init(argc, argv))
    return Usage();

  outfile.open(KnobOutputFile.Value().c_str(), std::ios_base::binary | std::ios_base::trunc);
  if (!outfile) {
    std::cout << "Couldn't open output trace file. Exiting." << std::endl;
    exit(1);
  }

  INS_AddInstrumentFunction(Instruction, 0);

  first_instr=KnobSkipInstructions.Value();
  last_instr=first_instr+KnobTraceInstructions.Value();
  
  PIN_AddFiniFunction(Fini, 0);

  PIN_MutexInit(&buffer_locks[0]);
  PIN_MutexInit(&buffer_locks[1]);
  PIN_MutexInit(&curr_instr_lock); // 初始化新增的锁
  PIN_SemaphoreInit(&sm);
  PIN_SpawnInternalThread((ROOT_THREAD_FUNC *)ActuallyWriteInstruction,NULL,0,&write_thid);
  
  PIN_StartProgram();

  return 0;
}

额外优化建议 #

• 将文件写入逻辑封装成独立函数，避免代码重复
• 为缓冲区预分配初始空间，减少push_back的内存分配开销
• 添加文件写入结果检查，便于调试定位IO错误

内容的提问来源于stack exchange，提问作者Sai Aravind