CMake配置PyArrow头文件后，Pybind11模块导入出现符号缺失错误

背景 #

Arrow CPP不再包含PyArrow头文件，这些头文件位于PyArrow安装目录（如${CONDA_PREFIX}/lib/python3.9/site-packages/pyarrow/include）。我配置CMakeLists.txt引入头文件，实现PyArrow Table在Python与C++间的类型转换，编译成功但导入模块时出现符号找不到的错误。

配置与代码 #

CMakeLists.txt配置 #

set(PYARROW_INCLUDE_DIR "${Python_SITELIB}/pyarrow/include")
set(PYARROW_INCLUDE_LOCAL_DIR "<HARDCODED_PATH>/lib/python3.9/site-packages/pyarrow/include")
IF(EXISTS ${PYARROW_INCLUDE_DIR})
    message(STATUS "Found and using pyarrow include dir: ${PYARROW_INCLUDE_DIR}")
    include_directories(${PYARROW_INCLUDE_DIR})
ELSEIF(EXISTS ${PYARROW_INCLUDE_LOCAL_DIR})
    message(STATUS "Found and using local development pyarrow include dir: ${PYARROW_INCLUDE_LOCAL_DIR}")
    include_directories(${PYARROW_INCLUDE_LOCAL_DIR})
#    target_include_directories(${PROJECT_NAME} PRIVATE ${PYARROW_INCLUDE_LOCAL_DIR})
ELSE()
    message(FATAL_ERROR "pyarrow include dir not found")
ENDIF() 

...

pybind11_add_module(${PROJECT_NAME} pyluban_cpp_ext.cpp ${SRCS_CODE})

target_link_libraries(${PROJECT_NAME} PRIVATE pybind11::module Arrow::arrow_shared GTest::gtest_main ${Boost_LIBRARIES})

target_include_directories(${PROJECT_NAME} PRIVATE ${PYARROW_INCLUDE_LOCAL_DIR})

类型转换代码 #

#include <arrow/python/pyarrow.h>
#include <Python.h>
#include <utility>
#include <memory>
#include <arrow/table.h>

namespace py = pybind11;

namespace PYBIND11_NAMESPACE { namespace detail {
        template <> struct type_caster<std::shared_ptr<arrow::Table>> {
        public:
        PYBIND11_TYPE_CASTER(std::shared_ptr<arrow::Table>, _("pyarrow::Table"));

        bool load(handle src, bool) {
            PyObject *source = src.ptr();
            if (!arrow::py::is_table(source))
                return false;
            arrow::Result<std::shared_ptr<arrow::Table>> result = arrow::py::unwrap_table(source);
            if(!result.ok())
                return false;
            value = result.ValueOrDie();
            return true;
        }

        static handle cast(std::shared_ptr<arrow::Table> src, return_value_policy /* policy */, handle /* parent */) {
            return arrow::py::wrap_table(src);
        }
    };
}} // namespace PYBIND11_NAMESPACE::detail

编译命令 #

rm -rf build
cmake -S . -B build
cmake --build build -j 2
cmake --install build --prefix $CONDA_PREFIX

错误信息 #

Traceback (most recent call last):
  File "<string>", line 1, in <module>
ImportError: dlopen(/project_name/build/pyluban_cpp_ext.cpython-39-darwin.so, 0x0002): symbol not found in flat namespace (__ZN5arrow2py10wrap_tableERKNSt3__110shared_ptrINS_5TableEEE)
Traceback (most recent call last):
  File "/project_name/build/test_script.py", line 1, in <module>
    from project_name import RollingTableManager
ImportError: dlopen(/project_name/build/project_name.cpython-39-darwin.so, 0x0002): symbol not found in flat namespace (__ZN5arrow2py10wrap_tableERKNSt3__110shared_ptrINS_5TableEEE)

PyArrow头文件目录结构 #

.
└── arrow
    ├── acero
    ├── adapters
    ├── api.h
    ├── array
    ├── array.h
    ├── buffer.h
    ├── buffer_builder.h
    ├── builder.h
    ├── c
    ├── chunk_resolver.h
    ├── chunked_array.h
    ├── compare.h
    ├── compute
    ├── config.h
    ├── csv
    ├── dataset
    ├── datum.h
    ├── device.h
    ├── engine
    ├── extension
    ├── extension_type.h
    ├── filesystem
    ├── flight
    ├── io
    ├── ipc
    ├── json
    ├── memory_pool.h
    ├── memory_pool_test.h
    ├── pch.h
    ├── pretty_print.h
    ├── python
    ├── record_batch.h
    ├── result.h
    ├── scalar.h
    ├── sparse_tensor.h
    ├── status.h
    ├── stl.h
    ├── stl_allocator.h
    ├── stl_iterator.h
    ├── table.h
    ├── table_builder.h
    ├── tensor
    ├── tensor.h
    ├── testing
    ├── type.h
    ├── type_fwd.h
    ├── type_traits.h
    ├── util
    ├── vendored
    ├── visit_array_inline.h
    ├── visit_data_inline.h
    ├── visit_scalar_inline.h
    ├── visit_type_inline.h
    ├── visitor.h
    └── visitor_generate.h

问题原因与解决方法 #

核心问题 #

你仅链接了Arrow::arrow_shared（Arrow核心C库），但arrow::py::wrap_table、arrow::py::unwrap_table这类函数属于PyArrow的C绑定部分，不在核心Arrow库中，而是存放在PyArrow的共享库内。编译时头文件可找到，但运行时无法定位对应符号。

解决步骤 #

1. 查找PyArrow共享库
   PyArrow共享库一般在其安装目录的lib子目录下，比如${CONDA_PREFIX}/lib/python3.9/site-packages/pyarrow/lib（macOS后缀为.dylib，Linux为.so）。

2. 在CMake中链接PyArrow共享库
   修改target_link_libraries配置，加入PyArrow共享库：

   # 自动查找PyArrow共享库
   find_library(PYARROW_LIBRARY
                NAMES pyarrow arrow_python
                PATHS "${Python_SITELIB}/pyarrow/lib"
                      "<HARDCODED_PATH>/lib/python3.9/site-packages/pyarrow/lib"
                NO_DEFAULT_PATH)
   
   if(NOT PYARROW_LIBRARY)
       message(FATAL_ERROR "PyArrow shared library not found")
   endif()
   
   # 更新链接配置
   target_link_libraries(${PROJECT_NAME} PRIVATE 
       pybind11::module 
       Arrow::arrow_shared 
       ${PYARROW_LIBRARY}  # 添加这一行
       GTest::gtest_main 
       ${Boost_LIBRARIES})
   

3. 确保运行时能加载PyArrow库

   • macOS/Linux：在CMake中设置运行路径，将PyArrow的lib目录加入模块的搜索路径：

     set_target_properties(${PROJECT_NAME} PROPERTIES
         INSTALL_RPATH "${Python_SITELIB}/pyarrow/lib"
         BUILD_RPATH "${Python_SITELIB}/pyarrow/lib")
     

   • 也可在运行Python时通过DYLD_LIBRARY_PATH（macOS）或LD_LIBRARY_PATH（Linux）指定PyArrow库路径。

4. 优化头文件引入方式
   将全局的include_directories改为仅针对目标的target_include_directories，避免污染全局编译环境：

   IF(EXISTS ${PYARROW_INCLUDE_DIR})
       message(STATUS "Found and using pyarrow include dir: ${PYARROW_INCLUDE_DIR}")
       target_include_directories(${PROJECT_NAME} PRIVATE ${PYARROW_INCLUDE_DIR})
   ELSEIF(EXISTS ${PYARROW_INCLUDE_LOCAL_DIR})
       message(STATUS "Found and using local development pyarrow include dir: ${PYARROW_INCLUDE_LOCAL_DIR}")
       target_include_directories(${PROJECT_NAME} PRIVATE ${PYARROW_INCLUDE_LOCAL_DIR})
   ELSE()
       message(FATAL_ERROR "pyarrow include dir not found")
   ENDIF()
   

   同时删除后面重复的target_include_directories(${PROJECT_NAME} PRIVATE ${PYARROW_INCLUDE_LOCAL_DIR})。

内容的提问来源于stack exchange，提问作者See